#!/usr/bin/env python
#-*- coding:utf-8 -*-
#author: Enoch time:2018/10/22 0031

import time
import re
import operator
from string import punctuation
import sys
from collections import Counter

def ProcessLine0(line,counts):

    line = re.sub('[^a-z]', '', line)
    for ch in line:
        counts[ch] = counts.get(ch, 0) + 1
    return counts

def CountLetter(path):
    file = open(path, 'r')
    wordsCount = 0
    alphabetCounts = {}
    for line in file:
        alphabetCounts = ProcessLine0(line.lower(), alphabetCounts)
    wordsCount = sum(alphabetCounts.values())
    alphabetCounts = sorted(alphabetCounts.items(), key=lambda k: k[0])
    alphabetCounts = sorted(alphabetCounts, key=lambda k: k[1], reverse=True)
    for letter, fre in alphabetCounts:
    	print("|\t{:15}|{:<11.2%}|".format(letter, fre / wordsCount))

    file.close()

def CountWords1(file_name,n,stopName,verbName):
    print("File name:" + sys.path[0] + "\\" + file_name)
    if (stopName != None):
        stopflag = True
    else:
        stopflag = False
    if(verbName != None):
        verbflag = True
    else:
        verbflag = False

    with open(file_name) as f:
        txt = f.read()
    txt = txt.lower()
    if(stopflag == True):
        with open(stopName) as f:
            stoplist = f.readlines()
    pattern = r"[a-z][a-z0-9]*"
    wordList = re.findall(pattern,txt)
    totalNum = len(wordList)
    tempc = Counter(wordList)
    if (stopflag == True):
        for word in stoplist:
            word = word.replace('\n','')
            del tempc[word]
    dicNum = dict(tempc.most_common(n))
    if (verbflag == True):
        totalNum = 0
        verbDic = {}
        verbDicNum = {}
        with open(verbName) as f:
            for line in f.readlines():
                key,value = line.split('')
                verbDic[key] = value.replace('\n','').split(',')
                verbDicNum[key] = tempc[key]
                for word in verbDic[key]:
                    verbDicNum[key] += tempc[word]
                totalNum += verbDicNum[key]
        verbDicNum = sorted(verbDicNum.items(), key=lambda k: k[0])
        verbDicNum = sorted(verbDicNum, key=lambda k: k[1], reverse=True)
    dicNum = sorted(dicNum.items(), key=lambda k:k[0])
    dicNum = sorted(dicNum, key=lambda k:k[1], reverse=True)

    if (verbflag == True):
        print(verbDicNum[:n])
    else:
        print(dicNum)

def CountPhrases(file_name,n,stopName,verbName,k):
    print("File name:" + sys.path[0] + "\\" + file_name)
    totalNum = 0
    if (stopName != None):
        stopflag = True
    else:
        stopflag = False
    if(verbName != None):
        verbflag = True
    else:
        verbflag = False

    with open(file_name) as f:
        txt = f.read()
    txt = txt.lower()
    txt = re.sub(r'[\s|\']+',' ',txt)
    pword = r'(([a-z]+ )+[a-z]+)'  # extract sentence
    pattern = re.compile(pword)
    sentence = pattern.findall(txt)
    txt = ','.join([sentence[m][0] for m in range(len(sentence))])
    if(stopflag == True):
        with open(stopName) as f:
            stoplist = f.readlines()
    pattern = "[a-z]+[0-9]*"
    for i in range(k-1):
        pattern += "[\s|,][a-z]+[0-9]*"
    wordList = []
    for i in range(k):
        if( i == 0 ):
            tempList = re.findall(pattern, txt)
        else:
            wordpattern = "[a-z]+[0-9]*"
            txt = re.sub(wordpattern, '', txt, 1).strip()
            tempList = re.findall(pattern, txt)
        wordList += tempList
    tempc = Counter(wordList)
    if (stopflag == True):
        for word in stoplist:
            word = word.replace('\n','')
            del tempc[word]
    dicNum = {}
    if (verbflag == True):
        verbDic = {}
        with open(verbName) as f:
            for line in f.readlines():
                key,value = line.split(' -> ')
                for tverb in value.replace('\n', '').split(','):
                    verbDic[tverb] = key
                verbDic[key] = key
        for phrase in tempc.keys():
            if (',' not in phrase):
                totalNum += 1
                verbList = phrase.split(' ')
                normPhrase = verbList[0]
                for verb in verbList[1:]:
                    if verb in verbDic.keys():
                        verb = verbDic[verb]
                    normPhrase += ' ' + verb
                if (normPhrase in dicNum.keys()):
                    dicNum[normPhrase] += tempc[phrase]
                else:
                    dicNum[normPhrase] = tempc[phrase]
    else:
        phrases = tempc.keys()
        for phrase in phrases:
            if (',' not in phrase):
                dicNum[phrase] = tempc[phrase]
                totalNum += tempc[phrase]
    dicNum = sorted(dicNum.items(), key=lambda k: k[0])
    dicNum = sorted(dicNum, key=lambda k: k[1], reverse=True)

    print(dicNum[:n])

软件工程/软件工程.py


import datetime
from typing import Any, Callable, Dict, Optional, Tuple, Union
from uuid import uuid4

from crontab import CronTab
from loguru import logger
from tortoise import timezone

from rearq.job import JobStatus
from rearq.server.models import Job, JobResult
from rearq.utils import ms_to_datetime, timestamp_ms_now, to_ms_timestamp

class Task:
    def __init__(
        self,
        bind: bool,
        function: Callable,
        queue: str,
        rearq,
        job_retry: int,
        job_retry_after: int,
        expire: Optional[Union[float, datetime.datetime]] = None,
    ):

        self.job_retry = job_retry
        self.job_retry_after = job_retry_after
        self.queue = queue
        self.rearq = rearq
        self.function = function
        self.bind = bind
        self.expire = expire

    async def delay(
        self,
        args: Optional[Tuple[Any, ...]] = None,
        kwargs: Optional[Dict[str, Any]] = None,
        job_id: str = None,
        countdown: Union[float, datetime.timedelta] = 0,
        eta: Optional[datetime.datetime] = None,
        expire: Optional[Union[float, datetime.datetime]] = None,
        job_retry: int = 0,
        job_retry_after: int = 60,
    ) -> Job:
        """
        Add job to queue.
        :param args: Job args.
        :param kwargs: Job kwargs.
        :param job_id: Custom job id.
        :param countdown: Delay seconds to execute.
        :param eta: Delay to datetime to execute.
        :param expire: Override default expire.
        :param job_retry: Override default job retry.
        :param job_retry_after: Override default job retry after.
        :return:
        """
        if not job_id:
            job_id = uuid4().hex
        if countdown:
            defer_ts = to_ms_timestamp(countdown)
        elif eta:
            defer_ts = to_ms_timestamp(eta)
        else:
            defer_ts = timestamp_ms_now()
        expire_time = None
        expires = expire or self.expire
        if expires:
            expire_time = ms_to_datetime(to_ms_timestamp(expires))

        job = await Job.get_or_none(job_id=job_id)
        if job:
            logger.warning(f"Job {job_id} exists")
            return job

        job = Job(
            task=self.function.__name__,
            args=args,
            kwargs=kwargs,
            job_retry=job_retry or self.job_retry,
            queue=self.queue,
            job_id=job_id,
            expire_time=expire_time,
            enqueue_time=timezone.now(),
            job_retry_after=job_retry_after,
        )

        if not eta and not countdown:
            job.status = JobStatus.queued
            await job.save()
            await self.rearq.redis.xadd(self.queue, {"job_id": job_id})
        else:
            job.status = JobStatus.deferred
            await job.save()
            await self.rearq.zadd(defer_ts, f"{self.queue}:{job_id}")

        return job

async def check_pending_msgs(self: Task, queue: str, group_name: str, timeout: int):
    """
    check pending messages
    :return:
    """
    redis = self.rearq.redis
    pending_msgs = await redis.xpending(self.queue, group_name, "-", "+", 10)
    p = redis.pipeline()
    execute = False
    for msg in pending_msgs:
        msg_id, _, idle_time, times = msg
        job_result = await JobResult.filter(msg_id=msg_id).only("job_id").first()
        if not job_result:
            continue
        if int(idle_time / 1000) > timeout * 2:
            execute = True
            p.xack(queue, group_name, msg_id)
            p.xadd(queue, {"job_id": job_result.job_id})
    if execute:
        return await p.execute()

class CronTask(Task):
    _cron_tasks: Dict[str, "CronTask"] = {}
    next_run: int

    def __init__(
        self,
        bind: bool,
        function: Callable,
        queue: str,
        rearq,
        job_retry: int,
        job_retry_after: int,
        cron: str,
        expire: Optional[Union[float, datetime.datetime]] = None,
        run_at_start: Optional[bool] = False,
    ):
        super().__init__(bind, function, queue, rearq, job_retry, job_retry_after, expire)
        self.crontab = CronTab(cron)
        self.cron = cron
        self.run_at_start = run_at_start
        self.set_next()

    def set_next(self):
        self.next_run = to_ms_timestamp(self.crontab.next(default_utc=False))

    @classmethod
    def add_cron_task(cls, function: str, cron_task: "CronTask"):
        cls._cron_tasks[function] = cron_task

    @classmethod
    def get_cron_tasks(cls):
        return cls._cron_tasks


import datetime as dt
import random
from functools import cached_property

from data import LOCATIONS, DATE_LOOKUP

class Date:
    def __init__(self, year, month, day, location=None, is_gregorian=True):
        self.year = year  # Year 0 is 1 BCE
        self.month = month
        self.day = day
        self.location = location
        self.is_gregorian = is_gregorian

    def __str__(self):
        result = self._date.strftime("%d. %B ")
        if self.year < 1:
            result += f"{abs(year) + 1} BCE"
        else:
            result += str(abs(self.year))
        if self.location:
            result += f" in {self.location}"
        return result

    @cached_property
    def _date(self):
        return dt.date(self.year if self.year > 0 else abs(self.year) + 1, self.month, self.day)

    @cached_property
    def is_leap_year(self):
        return is_leap_year(self.year, self.is_gregorian)

    @cached_property
    def weekday(self):
        """Monday is 0, Sunday is 6"""
        # we could use .weekday() plus an adjustment value down to the year 0, but
        # not before that, so let's just use the stupid table, I guess
        total = 0
        centuries = int(self.year / 100)
        years = self.year % 100
        if self.is_gregorian:
            total += DATE_LOOKUP["centuries_gregorian"][centuries % 4]
        else:
            total += DATE_LOOKUP["centuries_julian"][centuries % 7]
        total += DATE_LOOKUP["years"][years]
        if self.is_leap_year:
            total += DATE_LOOKUP["months_leap"][self.month]
        else:
            total += DATE_LOOKUP["months"][self.month]
        total += self.day
        return DATE_LOOKUP["weekday"][total % 7]

    @cached_property
    def weekday_string(self):
        return [
            "Monday",
            "Tuesday",
            "Wednesday",
            "Thursday",
            "Friday",
            "Saturday",
            "Sunday",
        ][self.weekday]

def get_random_date():
    """
    You'd think that grabbing a random date is easy, wouldn't you?

    First off, a qualifier: Picking a random /European/ date.
    This excludes non-European calendars, and calendar reforms
    in the colonies, because I can't be arsed to learn those.

    We're starting by picking a year and a month at random, because
    those always exist, thank fuck. Now, we can't pick a random
    day next, because the available days depend on the calendar
    in use, which, between 1582 and 1924, depends on the location.
    So we pick a location, and then we can make sure that
    a) we're not using leap days that did not exist, and b)
    that our random date did actually occur.

    I'm tempted to add a mode where "Trick question, this date did
    not occur" is a valid answer.

    Some of this is imprecise, because I can't be arsed to make
    sure countries are only included when they actually existed.
    """
    year = random.randrange(-44, dt.date.today().year + 200)
    month = random.randrange(1, 13)
    day = None
    location = None

    if (1582, 10) <= (year, month) <= (1924, 10):
        # uhhhm, where are we
        location = random.choice(list(LOCATIONS.keys()))
        julian_end, gregorian_start = LOCATIONS[location]

        if (year, month) < (julian_end[0], julian_end[1]):  # easy
            is_gregorian = False
        elif (year, month) > (gregorian_start[0], gregorian_start[1]):  # easy
            is_gregorian = True
        else:  # aw man
            if (
                year == julian_end[0] and month == julian_end[1]
            ):  # Biased towards Julian calendar, wooo
                is_gregorian = False
                day = random.randrange(1, julian_end[2] + 1)
            else:
                is_gregorian = True
                day = (
                    gregorian_start[2]
                    if gregorian_start[2] > 28
                    else random.randrange(gregorian_start[2], 29)
                )  # nothing to see here
    else:
        is_gregorian = year > 1800  # things could be so simple

    if not day:
        day = get_random_day(year, month, is_gregorian=is_gregorian)

    return Date(
        year=year, month=month, day=day, location=location, is_gregorian=is_gregorian
    )

def is_leap_year(year, is_gregorian):
    if is_gregorian:
        try:
            dt.date(year, 2, 29)
            return True
        except ValueError:
            return False
    return year % 4 == 0

def get_random_day(month, year, is_gregorian):
    if month != 2:  # We should be so lucky
        max_days = 30 if month in (4, 6, 9, 11) else 31
    else:
        max_days = 29 if is_leap_year(year, is_gregorian) else 28
    return random.randrange(1, max_days + 1)

def main():
    date = get_random_date()
    print(date)

if __name__ == "__main__":
    main()


from datetime import timedelta
import logging

from busy_beaver.clients import twitter
from busy_beaver.common.wrappers import KeyValueStoreClient, SlackClient
from busy_beaver.toolbox import utc_now_minus

LAST_TWEET_KEY = "last_posted_tweet_id"
logger = logging.getLogger(__name__)
kv_store = KeyValueStoreClient()

def fetch_tweets_post_to_slack(installation, channel_name, username):
    logger.info("Fetching tweets to post")
    tweets = get_tweets(installation, username)

    tweets_to_post = _exclude_tweets_inside_window(tweets, window=timedelta(minutes=30))

    logger.info("Grabbed {0} tweets".format(len(tweets_to_post)))
    # post 1 tweet at a time
    _post_to_slack(installation, channel_name, tweets_to_post[:1], username)

def get_tweets(installation, username):
    """Get latest tweets after last_posted_tweet_id"""
    tweets = twitter.get_user_timeline(username)
    last_posted_tweet_id = kv_store.get_int(installation.id, LAST_TWEET_KEY)
    recent_tweets = [tweet for tweet in tweets if tweet.id > last_posted_tweet_id]
    return list(reversed(recent_tweets))

def _exclude_tweets_inside_window(tweets, *, window: timedelta):
    """Buffer to delete tweets before retweeting to Slack"""
    boundary_dt = utc_now_minus(window)
    return [tweet for tweet in tweets if tweet.created_at <= boundary_dt]

def _post_to_slack(installation, channel_name, tweets, twitter_username):
    """Twitter Slack app unfurls URLs in Slack to show tweet details"""
    slack = SlackClient(installation.bot_access_token)

    url = "https://twitter.com/{username}/statuses/{id}"
    for tweet in tweets:
        tweet_url = url.format(username=twitter_username, id=tweet.id)
        slack.post_message(tweet_url, channel=channel_name)
        kv_store.put_int(installation.id, LAST_TWEET_KEY, tweet.id)


import string
import random
import os
import pandas as pd
from sklearn.model_selection import KFold
from sklearn.metrics import f1_score

import copy
import torch
import torch.optim as optim
import torch.nn as nn
import torchtext
from torchtext.vocab import Vectors
from torchtext.data import Dataset
import numpy as np
from torch.optim.optimizer import Optimizer
from functools import partial

# confing
SEED = 2021
random.seed(SEED)
user = "chizuchizu"
if user == "chizuchizu":
    BASE_PATH = '../for_train_data/'
else:
    BASE_PATH = "./data/"
TEXT_COL = "description"
TARGET = "jobflag"
NUM_CLASS = 4
N_FOLDS = 4
BS = 128
NUM_EPOCHS = 20
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

def seed_everything(seed):
    """for reproducibility.
    """
    random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.backends.cudnn.deterministic = True

seed_everything(SEED)

train = pd.read_csv(BASE_PATH + "train.csv").drop(['id'], axis=1)
train = train.rename(columns={TEXT_COL: 'text', TARGET: 'label'})
train['label'] -= 1

test = pd.read_csv(BASE_PATH + "test.csv").drop(['id'], axis=1)
test = test.rename(columns={TEXT_COL: 'text', TARGET: 'label'})

train.to_csv(BASE_PATH + 'train_x.csv', index=False, header=False)
test.to_csv(BASE_PATH + 'test_x.csv', index=False, header=False)

# print(train)

def preprocessing_text(text):
    for p in string.punctuation:
        text = text.replace(p, '')
    text = text.strip().split()
    return text

TEXT = torchtext.data.Field(sequential=True,
                            tokenize="spacy",
                            use_vocab=True,
                            batch_first=True,
                            include_lengths=True,
                            fix_length=128,
                            lower=True)

LABEL = torchtext.data.Field(sequential=False,
                             use_vocab=False)

# print(vars(train_ds[0]))

# 一回
# first = True
# if not os.path.isfile(BASE_PATH + "src/.vector_cache/wiki.en.vec"):
fasttext = torchtext.vocab.FastText(language="en")  # 分かち書きをvecotr化するここをfnとかにしたらフランス語に対応できるかも？
# else:
fasttext = Vectors(name='.vector_cache/wiki.en.vec')

'''
embeddingsはidでくるものをvectorにする。
'''

class LSTMClassifier(nn.Module):
    def __init__(self, text_id, hidden_dim, num_label):
        super(LSTMClassifier, self).__init__()
        self.gru_hidden_size = 64
        self.lstm_hidden_size = 300
        self.embeddings = nn.Embedding.from_pretrained(
            embeddings=text_id, freeze=True
        )
        self.embedding_dropout = nn.Dropout2d(0.2)
        self.lstm = nn.LSTM(self.lstm_hidden_size, hidden_dim, batch_first=True, bidirectional=True)

        self.gru = nn.GRU(hidden_dim * 2, self.gru_hidden_size, bidirectional=True, batch_first=True)
        self.linear = nn.Linear(self.gru_hidden_size * 6, 20)
        self.cls = nn.Linear(hidden_dim, num_label)

        self.relu = nn.ReLU()
        self.dropout = nn.Dropout(0.1)
        self.out = nn.Linear(self.gru_hidden_size * 6, num_label)
        # self.softmax = nn.LogSoftmax()

    def apply_spatial_dropout(self, h_embedding):
        h_embedding = h_embedding.transpose(1, 2).unsqueeze(2)
        h_embedding = self.embedding_dropout(h_embedding).squeeze(2).transpose(1, 2)
        return h_embedding

    def forward(self, x):
        x_vec = self.embeddings(x)
        x_vec = self.apply_spatial_dropout(x_vec)

        h_lstm, lstm_out = self.lstm(x_vec)
        h_gru, hh_gru = self.gru(h_lstm)
        hh_gru = hh_gru.view(-1, self.gru_hidden_size * 2)

        avg_pool = torch.mean(h_gru, 1)
        max_pool, _ = torch.max(h_gru, 1)
        conc = torch.cat((hh_gru, avg_pool, max_pool), 1)

        # conc = self.linear(conc)

        # conc = self.relu(conc)
        conc = self.dropout(conc)

        out = self.out(conc)
        return out

def metric_f1(labels, preds):
    return f1_score(labels, preds, average='macro')

class EMA(nn.Module):

    def __init__(self, model, mu, level='batch', n=1):
        # self.ema_model = copy.deepcopy(model)
        super(EMA, self).__init__()
        self.mu = mu
        self.level = level
        self.n = n
        self.cnt = self.n
        self.shadow = {}
        for name, param in model.named_parameters():
            if param.requires_grad:
                self.shadow[name] = param.data

    def _update(self, model):
        for name, param in model.named_parameters():
            if param.requires_grad:
                new_average = (1 - self.mu) * param.data + self.mu * self.shadow[name]
                self.shadow[name] = new_average.clone()

    def set_weights(self, ema_model):
        for name, param in ema_model.named_parameters():
            if param.requires_grad:
                param.data = self.shadow[name]

    def on_batch_end(self, model):
        if self.level is 'batch':
            self.cnt -= 1
            if self.cnt == 0:
                self._update(model)
                self.cnt = self.n

    def on_epoch_end(self, model):
        if self.level is 'epoch':
            self._update(model)

class ParamScheduler:

    def __init__(self, optimizer, scale_fn, step_size):
        if not isinstance(optimizer, Optimizer):
            raise TypeError('{} is not an Optimizer'.format(
                type(optimizer).__name__))

        self.optimizer = optimizer
        self.scale_fn = scale_fn
        self.step_size = step_size
        self.last_batch_iteration = 0

    def batch_step(self):
        for param_group in self.optimizer.param_groups:
            param_group['lr'] = self.scale_fn(self.last_batch_iteration / self.step_size)

        self.last_batch_iteration += 1

def combine_scale_functions(scale_fns, phases=None):
    if phases is None:
        phases = [1. / len(scale_fns)] * len(scale_fns)
    phases = [phase / sum(phases) for phase in phases]
    phases = torch.tensor([0] + phases)
    phases = torch.cumsum(phases, 0)

    def _inner(x):
        idx = (x >= phases).nonzero().max()
        actual_x = (x - phases[idx]) / (phases[idx + 1] - phases[idx])
        return scale_fns[idx](actual_x)

    return _inner

def scale_cos(start, end, x):
    return start + (1 + np.cos(np.pi * (1 - x))) * (end - start) / 2

def eval_model(model, data_loader, is_train=False):
    all_labels = []
    all_preds = []
    outputs_ = []
    for batch in data_loader:
        inputs = batch.text[0].to(device)
        if is_train:
            labels = batch.label.to(device)
        with torch.set_grad_enabled(False):
            outputs = model(inputs)
            _, pred = torch.max(outputs, 1)
            if is_train:
                all_labels += labels.tolist()
            all_preds += pred.tolist()
            outputs_ += outputs.tolist()

    return all_labels, all_preds, np.array(outputs_)

def train_model(model, dl_dict, criterion, optimizer, num_epochs):
    model.to(device)
    ema_model = copy.deepcopy(model)
    ema_model.eval()
    ema_n = int(len(dl_dict['train'].dataset) / (5 * BS))
    ema = EMA(model, 0.9, n=ema_n)
    scale_fn = combine_scale_functions(
        [partial(scale_cos, 1e-4, 5e-3), partial(scale_cos, 5e-3, 1e-3)], [0.2, 0.8])
    scheduler = ParamScheduler(optimizer, scale_fn, num_epochs * len(dl_dict["train"]))
    all_test_preds = list()
    all_oof_preds = list()
    all_oof_preds_ema = list()
    all_test_output = list()
    all_oof_output = list()
    for epoch in range(num_epochs):
        all_loss = 0
        # all_labels = []
        # all_preds = []
        for batch in dl_dict['train']:
            inputs, inputs_length = batch.text[0].to(device), batch.text[1].to(device)  # 文章
            labels = batch.label.to(device)
            optimizer.zero_grad()

            ema.on_batch_end(model)

            scheduler.batch_step()

            with torch.set_grad_enabled(True):
                outputs = model(inputs)
                loss = criterion(outputs, labels)
                _, preds = torch.max(outputs, 1)
                loss.backward()
                optimizer.step()
                all_loss += loss.item()
        print("train | epoch", epoch + 1, " | ", "loss", all_loss / len(dl_dict["train"]))
        all_labels, all_preds, output_pred = eval_model(model, dl_dict["val"], is_train=True)
        all_oof_preds.append(all_preds)
        all_oof_output.append(output_pred)
        train_f1 = f1_score(all_labels, all_preds, average="macro")
        print("val | epoch", epoch + 1, " | ", "f1", train_f1)

        test_labels, test_preds, test_output = eval_model(model, dl_dict["test"], is_train=False)
        all_test_preds.append(test_preds)
        all_test_output.append(test_output)
        # all_test_preds.append(eval_model(model, dl_dict["test"], is_train=False)[1])

    ema.set_weights(ema_model)
    ema_model.lstm.flatten_parameters()
    ema_model.gru.flatten_parameters()
    all_labels_ema, all_preds_ema, output_pred_ema = eval_model(ema_model, dl_dict["val"], is_train=True)
    all_oof_preds_ema.append(all_preds_ema)
    # all_oof_output_ema.append(output_pred_ema)
    ema_f1 = f1_score(all_labels_ema, all_preds_ema, average="macro")
    print('ema f1', ema_f1)

    test_labels_ema, test_preds_ema, output_test_ema = eval_model(ema_model, dl_dict["test"], is_train=False)

    checkpoint_weights = np.array([3 ** epoch for epoch in range(num_epochs)])
    checkpoint_weights = checkpoint_weights / checkpoint_weights.sum()
    # eva/
    # test_y = np.average(all_test_preds, weights=checkpoint_weights, axis=0).astype(float)
    # oof = np.average(all_oof_preds, weights=checkpoint_weights, axis=0).astype(float)

    output_y = np.average(all_test_output, weights=checkpoint_weights, axis=0).astype(float)
    output_oof = np.average(all_oof_output, weights=checkpoint_weights, axis=0).astype(float)

    # test_y = np.mean([test_y, test_preds_ema], axis=0)
    # oof = np.mean([oof, all_oof_preds_ema], axis=0)

    test_y = np.mean([output_y, output_test_ema], axis=0)
    oof = np.mean([output_oof, output_pred_ema], axis=0)
    # test_y = np.array(test_preds_ema)
    # oof = np.array(all_oof_preds_ema)
    # test_y = np.round(test_y).astype(int)

    # test_y = np.mean([])

    return ema_model, model, test_y, oof

train_ds, test_ds = torchtext.data.TabularDataset.splits(
    path=BASE_PATH, train='train_x.csv',
    test='test_x.csv', format='csv',
    fields=[('text', TEXT), ('label', LABEL)]
)
test_ds = torchtext.data.TabularDataset(
    path=BASE_PATH + "test_x.csv",
    format="csv",
    fields=[("text", TEXT)],
)
TEXT.build_vocab(train_ds, vectors=fasttext, min_freq=3)  # buildしないといけないらしいよくわからない
TEXT.build_vocab(test_ds, vectors=fasttext, min_freq=3)
kf = KFold(n_splits=4, shuffle=True, random_state=SEED)
y_pred = np.zeros((test.shape[0], 4))
oof = np.zeros((train.shape[0], 4))
for fold, (tdx, vdx) in enumerate(kf.split(train_ds.examples)):
    print(fold + 1)
    data_arr = np.array(train_ds.examples)

    train_dl = torchtext.data.Iterator(Dataset(data_arr[tdx], fields=[("text", TEXT), ("label", LABEL)]), batch_size=BS,
                                       train=True)
    val_dl = torchtext.data.Iterator(Dataset(data_arr[vdx], fields=[("text", TEXT), ("label", LABEL)]), batch_size=BS,
                                     train=False, sort=False)
    test_dl = torchtext.data.Iterator(test_ds, batch_size=BS, train=False, sort=False)
    dl_dict = {'train': train_dl, 'val': val_dl, 'test': test_dl}

    model = LSTMClassifier(TEXT.vocab.vectors, 128, NUM_CLASS)
    # 損失関数
    weight = len(train) / train["label"].value_counts().sort_index().values
    weights = torch.tensor(weight.tolist()).to(device)
    criterion = nn.CrossEntropyLoss(weight=weights)
    # criterion = nn.CrossEntropyLoss()
    # オプティマイザー
    optimizer = optim.Adam(model.parameters(), lr=0.01)

    ema_model, model, test_y, oof_ = train_model(model, dl_dict, criterion, optimizer, NUM_EPOCHS)
    # model_path = f"../models/lstm/ema_{fold}.pth"
    # torch.save(ema_model.state_dict(), model_path)
    # model_path = f"../models/lstm/lstm_{fold}.pth"
    # torch.save(model.state_dict(), model_path)

    y_pred += test_y / N_FOLDS
    oof[vdx, :] = oof_  # np.round(oof_.astype(float).astype(float)).astype(int)

    # y_pred = np.round(y_pred.astype(float)).astype(int)
    model_path = f"../models/lstm/ema_{fold}.pth"

language = "default"
columns = ["0", "1", "2", "3"]

lang_columns = [language + "_lstm_" + x for x in columns]

test_pred = pd.DataFrame(y_pred)
oof_pred = pd.DataFrame(oof)

test_pred.columns = lang_columns
oof_pred.columns = lang_columns
test_pred.to_csv(f"../for_train_data/lstm/test_{language}_lstm.csv", index=False)
oof_pred.to_csv(f"../for_train_data/lstm/train_{language}_lstm.csv", index=False)
# def make_submit_file(pred):
#     test_id = pd.read_csv(BASE_PATH + "test.csv")["id"]
#     submit = pd.DataFrame({'index': test_id, 'pred': pred + 1})
#     # aug = "using_aug" if augmentation else "non_aug"
#     submit.to_csv(f"../outputs/lstm_v1.csv", index=False, header=False)
#
# make_submit_file(y_pred)

print("DONE")


#-*- coding: utf-8 -*-
import pickle
import time
from copy import deepcopy

from flask import request, redirect
from flask.helpers import make_response
from sqlalchemy.sql.expression import text, func, asc
from sqlalchemy.sql.functions import sum
from traffgroup.core.model.billing.mtsubscription import MTSubscription
from traffgroup.core.model.billing.requestlog import RequestLog
from traffgroup.core.model.meta import Session
from traffgroup.core.model.statistics import Stat
from traffgroup.core.X.mako import render_template
from traffgroup.core.X.utils import XRedis
from traffgroup.core.X.xflask import Controller, Route, g, capp
from traffgroup.x.security import CProtect
from traffgroup.x.util.timestamp import TimeStamp
from traffgroup.admin.lib.predicate import IsAdmin

# generic stats interval
GEN_STATS_DAYS = 30

# a week ago
get_ts = lambda: TimeStamp.today_tz() + time.timezone - GEN_STATS_DAYS * TimeStamp.DAYSECONDS

# end of current day
get_te = lambda: TimeStamp.today_tz() + time.timezone + TimeStamp.DAYSECONDS

@CProtect(IsAdmin())
@Controller
class IndexController(object):

    @staticmethod
    def _get_sub_ids(ts, force=False):
        """
        @returns subscriptions ids for a current day
        """
        xredis = XRedis(db=2)
        sub_ids = xredis.get('main_sub_ids_%s' % ts)
        if sub_ids:
            try:
                sub_ids = pickle.loads(sub_ids)
            except Exception:
                # failed to unpickle
                force = True

        if not sub_ids or force:
            subs = MTSubscription.Filter(
                MTSubscription.ts_spawn >= ts,
                MTSubscription.ts_spawn < ts + TimeStamp.DAYSECONDS - 1,
                MTSubscription.state != MTSubscription.State.INITIAL,
                MTSubscription.type == MTSubscription.Type.REGULAR,
                MTSubscription.operator.in_([1, 2, 3]))
            sub_ids = [x.id for x in subs.all()]
            if ts != TimeStamp.today_tz() + time.timezone:
                # store for a week
                xredis.setex('main_sub_ids_%s' % ts, 604800,
                             pickle.dumps(sub_ids))
        return sub_ids

    @staticmethod
    def _get_dyn(ts, te, only_last=False, from_ts=None, force=False):
        day_secs = TimeStamp.DAYSECONDS

        # subscriptions ids
        subs_ids = IndexController._get_sub_ids(ts, force=force)

        daily_preset = ((RequestLog.ts_spawn - time.timezone) -
                        ((RequestLog.ts_spawn - time.timezone) % day_secs))
        time_start = ts if not only_last else from_ts
        db_data = (Session.query(func.count(RequestLog.id).label('cnt'),
                                 daily_preset.label('daily_spawn'),
                                 RequestLog.type,
                                 RequestLog.operator)
                   .filter(RequestLog.ts_spawn >= time_start,
                           RequestLog.ts_spawn < te,
                           RequestLog.state == RequestLog.State.OK,
                           RequestLog.ref_id.in_(subs_ids),
                           RequestLog.operator.in_([1, 2, 3]),
                           RequestLog.type.in_([RequestLog.Type.NOTIFY_SUB,
                                                RequestLog.Type.REBILL,
                                                RequestLog.Type.NOTIFY_UNSUB]))
                   .group_by(text('daily_spawn'),
                             RequestLog.type,
                             RequestLog.operator).all())

        stat = {}
        default_row = {1: {'subs': 0, 'rebills': 0, 'unsubs': 0},
                       2: {'subs': 0, 'rebills': 0, 'unsubs': 0},
                       3: {'subs': 0, 'rebills': 0, 'unsubs': 0}}

        drkeys = {RequestLog.Type.NOTIFY_SUB: 'subs',
                  RequestLog.Type.REBILL: 'rebills',
                  RequestLog.Type.NOTIFY_UNSUB: 'unsubs'}

        for d in db_data:
            if d.daily_spawn not in stat:
                stat[d.daily_spawn] = deepcopy(default_row)
            stat[d.daily_spawn][d.operator][drkeys[d.type]] += d.cnt

        total = deepcopy(default_row)
        for ts_spawn, row in stat.items():
            for operator, values in row.items():
                total[operator]['subs'] += values['subs']
                total[operator]['unsubs'] += values['unsubs']
                total[operator]['rebills'] += values['rebills']

        return stat, total

    @staticmethod
    def _get_daily_stats(ts, te):
        q = (Session.query((Stat.Presets.GROUP_DAILY).label('ts_spawn'),
                           sum(Stat.subs).label('subs'),
                           sum(Stat.unsubs).label('unsubs'),
                           sum(Stat.rebills).label('rebills'))
             .filter(Stat.ts_spawn >= ts, Stat.ts_spawn < te)
             .group_by(Stat.Presets.GROUP_DAILY)
             .order_by(asc(text('ts_spawn'))))
        stats = q.all()
        return stats

    @Route("/", methods=['GET'])
    def index(self):
        if 'no_redirect' not in request.values:
            return redirect('/stats/?group=hourly')

        xredis = XRedis(db=2)
        ts = get_ts()
        te = get_te()
        force = 'force' in request.values
        need_cache_stats = False

        # fetch yesterday dynamics
        dyn, total_dyn = self._get_dyn(TimeStamp.today_tz() + time.timezone -
                                       TimeStamp.DAYSECONDS, te, force=force)
        if len(total_dyn) == 3:
            subs = (total_dyn[1]['subs'] +
                    total_dyn[2]['subs'] +
                    total_dyn[3]['subs'])
            g.unsubs_percent = (((total_dyn[1]['unsubs'] +
                                  total_dyn[2]['unsubs'] +
                                  total_dyn[3]['unsubs']) /
                                 float(subs) * 100) if subs else 0)
            g.rebills_percent = (((total_dyn[1]['rebills'] +
                                   total_dyn[2]['rebills'] +
                                   total_dyn[3]['rebills']) /
                                  float(subs) * 100) if subs else 0)
        else:
            g.unsubs_percent = g.rebills_percent = 0.0

        # current convert calculation
        stats = (Session.query(sum(Stat.unique).label('unique'),
                               sum(Stat.subs).label('subs'),
                               sum(Stat.pseudo).label('pseudo'))
                 .filter(Stat.ts_spawn >= TimeStamp.today_tz() + time.timezone)
                 .filter(Stat.ts_spawn < TimeStamp.today_tz() + time.timezone +
                                         TimeStamp.DAYSECONDS - 1)).all()
        stat = stats[0]
        g.current_convert = (int(stat.unique / (stat.subs + stat.pseudo))
                             if stat.subs is not None and
                                stat.pseudo is not None and
                                (stat.subs + stat.pseudo) else 0)

        # get stats
        generic_stats = []
        if not force:
            # from cache
            generic_stats = xredis.get('admin_generic_stats') or {}
            if generic_stats:
                try:
                    generic_stats = pickle.loads(generic_stats)
                except Exception:
                    # unpickling filed
                    force = True
        if force or not len(generic_stats):
            # from db
            capp.logger.info('fetch generic stats from db')
            generic_stats = self._get_daily_stats(ts, te)
            need_cache_stats = True
        else:
            last_day = max([s.ts_spawn for s in generic_stats]) + time.timezone
            today = TimeStamp.today_tz() + time.timezone
            if last_day <= today:
                # cached stats out of date
                capp.logger.info('cached stats out of date')
                # remove last days
                generic_stats = generic_stats[: -((today - last_day) /
                                                  TimeStamp.DAYSECONDS + 1)]
                if last_day == today:
                    generic_stats += self._get_daily_stats(last_day, te)
                else:
                    generic_stats += self._get_daily_stats(
                        last_day - TimeStamp.DAYSECONDS * 2, te)
                # remove old days
                for stat in generic_stats:
                    if stat.ts_spawn < ts:
                        capp.logger.info('remove old day')
                        del generic_stats[generic_stats.index(stat)]
                        need_cache_stats = True
                    else:
                        break
                if last_day != TimeStamp.today_tz() + time.timezone:
                    need_cache_stats = True
        if need_cache_stats:
            # store in cache for an interval
            capp.logger.info('store in cache for an interval')
            xredis.setex('admin_generic_stats',
                         GEN_STATS_DAYS * TimeStamp.DAYSECONDS,
                         pickle.dumps(generic_stats))

        g.stats = generic_stats

        return render_template('index/index.mako')

    @Route("/make_error")
    def error(self):
        return make_response('some error', 500)


#!/usr/bin/env python

import argparse
import csv
import urllib2

from cStringIO import StringIO
from dateutil.parser import parse
from pykronos import KronosClient
from pykronos import TIMESTAMP_FIELD
from zipfile import ZipFile

"""
Loads US presidential elections contribution data for the 2012 presidential
race in the state of Oregon from the US Federal Election Commission into
Kronos.

A sample event dictionary looks like:

{u'@id': u'9809c000-17ed-11e2-8000-0b89d16a9975',
 u'@time': 13504320000000000L,
 u'cand_id': u'P80003338',
 u'cand_nm': u'Obama, Barack',
 u'cmte_id': u'C00431445',
 u'contb_receipt_amt': u'6',
 u'contb_receipt_dt': u'17-OCT-12',
 u'contbr_city': u'PORTLAND',
 u'contbr_employer': u'NOT EMPLOYED',
 u'contbr_nm': u'BROOKS, MAGGIE',
 u'contbr_occupation': u'STUDENT',
 u'contbr_st': u'OR',
 u'contbr_zip': u'972171333',
 u'election_tp': u'G2012',
 u'file_num': u'897092',
 u'form_tp': u'SA17A',
 u'memo_cd': u'',
 u'memo_text': u'',
 u'null': [u''],
 u'receipt_desc': u'',
 u'tran_id': u'C26603850'}
"""

DONATIONS_FILE_NAME = 'P00000001-OR'
DONATIONS_FILE_URL = ('ftp://ftp.fec.gov/FEC/Presidential_Map/2012/P00000001/'
                      '%s.zip' % DONATIONS_FILE_NAME)

def load_test_data(args):
  donations = ZipFile(StringIO(urllib2.urlopen(DONATIONS_FILE_URL).read()))
  donations = StringIO(donations.read('%s.csv' % DONATIONS_FILE_NAME))

  events = []
  rows = csv.DictReader(donations)
  for row in rows:
    row[TIMESTAMP_FIELD] = parse(row['contb_receipt_dt'])
    events.append(row)

  kc = KronosClient(args.kronos_url)
  kc.put({'donations': events})

if __name__ == '__main__':
  parser = argparse.ArgumentParser()
  parser.add_argument('--kronos-url',
                      default='http://localhost:8150',
                      help='The Kronos server to dump data into')
  args = parser.parse_args()
  load_test_data(args)


import cv2
import numpy
import random

def imtoArrGrayScaleConverter(image):
    ArrayG=cv2.imread(image,0)
    return ArrayG

def imtoArrColorConverter(image):
    ArrayC=cv2.imread(image,1)
    return ArrayC

def ArrtoImConverter(Array):
    newImageName=input("Enter new image name with extension: ")
    Image=cv2.imwrite(newImageName,Array)
    return Image

def ArrayGenerator():
    List1=[]

    for i in range(0,64):
        List1.append(random.randint(0,255))

    Array=numpy.asarray(List1)
    FinalArray=Array.reshape(4,4,4)

    return FinalArray

def menu():

    print("MAIN MENU")
    print("==========")

    print("1. Convert Image to array (Grayscale) ")
    print("2. Convert Image to array (RGB)")
    print("3. Computer Generated Image (Small Dimensions in V.01 of this program)")

def main():

    menu()

    userInput=int(input("Enter One of the Above (1-3): "))

    if userInput==1:
        ImageName=input("Enter Image File Title With Extension: ")
        GrayImage=imtoArrGrayScaleConverter(ImageName)
        print(GrayImage)

    elif userInput==2:
        ImageName=input("Enter Image File Title With Extension: ")
        ColorImage=imtoArrColorConverter(ImageName)
        print(ColorImage)

    elif userInput==3:
        Array=ArrayGenerator()
        Image=ArrtoImConverter(Array)
        print(Image)

main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import unittest
from mock import patch, DEFAULT, call, Mock
import six

from mylinux.view.tuiElement import module_CreateApp
from mylinux.view.tuiElement import CreateApp

from mylinux.libs.Assert import Assert

Test = Assert()

class Class_CreateApp(unittest.TestCase):
	def setUp(self):
		self.Class = CreateApp
		self.instance = self.Class()

		self.patch_module = patch.multiple(
			module_CreateApp,
			npyscreen=DEFAULT,
			CreateForm=DEFAULT,
			AppErr=DEFAULT
		)

		self.mock_module = self.patch_module.start()
	def tearDown(self):
		self.patch_module.stop()

	def test_COVERAGE(self):
		Test.coverage(
			typeElements=[self.Class, self.instance],
			testClass=Class_CreateApp,
			testClassSkiped=['tearDown', 'setUp', 'test_COVERAGE', '__module__', '__doc__', '_classSetupFailed'],
			typeRemove=[],
			typeSkiped=['__doc__','__module__','NEXT_ACTIVE_FORM','_Forms','_FORM_VISIT_LIST','_LAST_NEXT_ACTIVE_FORM'],
		)

	def test_requiredModelValues(self):
		self.assertListEqual(
			self.Class.requiredModelValues,
			[
				'hints',
				'packageName',
				'package-install',
				'package-purge',
				'config-install',
				'config-purge',
				'test.py',
			]
		)

	def test_onStart(self):
		self.instance.modelValues = {
			'hints' : 'hints.value',
			'packageName' : 'packageName.value',
			'package-install':'package-install.value',
			'package-purge' : 'package-purge.value',
			'config-install' : 'config-install.value',
			'config-purge' : 'config-purge.value',
			'test.py' : 'test.py.value'
		}

		self.mock_module['CreateForm'].return_value = Mock()

		with patch.multiple(self.instance,
			setModelValues=DEFAULT,
			checkModelValues=DEFAULT,
			registerForm=DEFAULT
		) as mock_self:

			self.instance.onStart()

		mock_self['setModelValues'].assert_called_once_with()
		mock_self['checkModelValues'].assert_called_once_with()

		self.mock_module['CreateForm'].assert_called_once_with(
			lines=40,
			name='Create package: packageName.value',
			columns=80
		)
		self.assertEqual(
			self.instance.mainForm.packageInstall.entry_widget.values,
			'package-install.value'
		)
		self.assertEqual(
			self.instance.mainForm.packagePurge.entry_widget.values,
			'package-purge.value'
		)
		self.assertEqual(
			self.instance.mainForm.configInstall.entry_widget.values,
			'config-install.value'
		)
		self.assertEqual(
			self.instance.mainForm.configPurge.entry_widget.values,
			'config-purge.value'
		)
		self.assertEqual(
			self.instance.mainForm.test.entry_widget.values,
			'test.py.value'
		)
		mock_self['registerForm'].assert_called_once_with(
			'MAIN',
			self.mock_module['CreateForm']()
		)

	def test_onCleanExit(self):
		mock_mainForm = Mock()
		mock_mainForm.packageInstall.entry_widget.values = ['PI','PI1','PI2']
		mock_mainForm.packagePurge.entry_widget.values = ['PP','PP1','PP2']
		mock_mainForm.configInstall.entry_widget.values = ['CI','CI1','CI2']
		mock_mainForm.configPurge.entry_widget.values = ['CP','CP1','CP2']
		mock_mainForm.test.entry_widget.values = ['T','T1','T2']

		self.instance.mainForm = mock_mainForm

		with patch.object(self.instance,'exit') as mock_exit:

			self.instance.onCleanExit()

		mock_exit.assert_called_once_with(
			info={
				'info': mock_mainForm.info.value,
				'class': mock_mainForm.className.value,
				'module':mock_mainForm.moduleName.value
			},
			scripts={
				'test.py': 'T\nT1\nT2',
				'config-install': 'CI\nCI1\nCI2',
				'package-purge': 'PP\nPP1\nPP2',
				'config-purge': 'CP\nCP1\nCP2',
				'package-install': 'PI\nPI1\nPI2'
			}
		)

	def test_checkModelValues(self):

		self.instance.modelValues = {
			'hints' : 'value',
			'packageName' : 'value',
			'package-install' : 'value',
			'package-purge' : 'value',
			'config-install' : 'value',
			'config-purge' : 'value',
			'test.py' : 'value'
		}

		self.instance.checkModelValues()

	def test_checkModelValues_missingKeyInModelValues(self):
		self.instance.modelValues = {}

		self.mock_module['AppErr'].developer.side_effect = Exception('AppErr.developer.side_effect')

		with self.assertRaises(Exception) as err:

			self.instance.checkModelValues()

		errMsg = Assert.getErrorMessage(err)

		self.mock_module['AppErr'].developer.assert_called_once_with(
			"Missing in modelValues! ==> ['hints', 'packageName', 'package-install', 'package-purge', 'config-install', 'config-purge', 'test.py']"
		)
		self.assertEqual(
			errMsg,
			'AppErr.developer.side_effect'
		)

	def test_setModelValues(self):
		self.instance.setModelValues()

	def test_exit(self):
		self.instance.exit(test='test')

if __name__ == '__main__':
	unittest.main()


#-*- coding: utf-8

from ihna.kozhukhov.core import ProcessedData
from MapTable import MapTable
from FilteredMapTable import FilteredMapTable
from Hist1DDataManager import Hist1DDataManager

import numpy as np
import matplotlib.pyplot as plt

class Hist1D(ProcessedData):
	'''
	prints a one-dimensional histogram
	'''

	def getName(self):
		'''
		Returns a histogram
		'''
		return "hist"

	def setSource(self, value):
		'''
		Sets a source
		'''
		if isinstance(value, MapTable):
			super(Hist1D, self).setSource(value)
		else:
			raise ValueError("{0} is not a valid source for the object".format(value.__class__.__name__))

	def __str__(self):
		S = super(Hist1D, self).__str__()+"\n"
		try:
			S += "Binning value: {0}\n".format(self.getBinningValue())
			S += "Binning edges: "
			for x in self.getBinEdges():
				S += "{0:.2f}\t".format(x)
		except AttributeError as err:
			S += "\033[31m{0}\033[0m".format(err)
		return S

	__binningValue = None

	def getBinningValue(self):
		'''
		Returns a histogram value
		'''
		if self.__binningValue is None:
			raise AttributeError("Please, specify a binning value by setBinningValue")
		return self.__binningValue

	def setBinningValue(self, value):
		'''
		Sets a binning value
		'''
		if not isinstance(self.getSource(), Hist1DDataManager) and not self.getSource() is None:
			idx = self.getSource().getColumns().index(value)
		self.__binningValue = value

	def isCircular(self):
		'''
		Returns True if the binning value is circular
		'''
		return self.getBinningValue().find("orientation")!=-1

	__binEdges = None

	def getBinEdges(self):
		'''
		Returns bin centers
		'''
		if self.__binEdges is None:
			raise AttributeError("Please, specify bin edges by setBinEdges or \
bin size by setBinRange or by setBinNumber")
		return self.__binEdges

	def setBinEdges(self, value):
		'''
		Sets bin centers
		'''
		if isinstance(value, np.ndarray):
			self.__binEdges = value
		else:
			raise ValueError("Please, specify bin edges")

	def setBinNumber(self, N):
		'''
		Sets bin number (for circular values only)
		'''
		if not self.isCircular():
			raise ValueError("The function is applicable for circular binning data only")
		binSize = np.pi/int(N)
		leftBorder = -np.pi/2-binSize/2
		rightBorder = np.pi/2+binSize/2
		edges = np.arange(leftBorder, rightBorder+1e-10, binSize)
		self.setBinEdges(edges)

	def setBinSize(self, left, size, right):
		'''
		Sets the bin size (for linear data only)

		left - left border of the data
		right - right border of the data
		size - the bin size
		'''
		if self.isCircular():
			raise ValueError("The function is applicable for linear data only")
		edges = np.arange(left, right+1e-10, size)
		self.setBinEdges(edges)

	def getBinCenters(self):
		'''
		Returns bin centers
		'''
		edges = self.getBinEdges()
		return 0.5*(edges[:-1]+edges[1:])

	__columns = None

	def getColumns(self):
		'''
		Returns a list of all columns
		'''
		if self.__columns is None:
			raise ValueError("The data was not processed")
		return self.__columns

	__counts = None

	def getCounts(self):
		'''
		Returns counts
		'''
		if not self.isDataProcessed():
			raise ValueError("The data was not processed")
		return self.__counts

	def getProportions(self):
		'''
		Returns proportions
		'''
		counts = self.getCounts()
		return counts/counts.sum()

	def getPercentages(self):
		'''
		Returns percentages
		'''
		return self.getProportions()*100

	__values = None
	__meanValues = None
	__stdValues = None
	__sterrValues = None

	def getValues(self, colname, centerIdx):
		'''
		get column values for a certain column name and a certain center index
		'''
		if not self.isDataProcessed():
			raise ValueError("Data was not processed")
		return self.__values[colname][centerIdx]

	def getMeanValues(self, colname=None):
		'''
		get mean values for the column
		'''
		if colname is None:
			return self.__meanValues
		if not self.isDataProcessed():
			raise ValueError("Data was not processed")
		col = self.getColumns().index(colname)
		return self.__meanValues[:,col]

	def getStdValues(self, colname=None):
		'''
		get mean values for the column
		'''
		if colname is None:
			return self.__stdValues
		if not self.isDataProcessed():
			raise ValueError("Data was not processed")
		col = self.getColumns().index(colname)
		return self.__stdValues[:,col]

	def getSterrValues(self, colname=None):
		'''
		get mean values for the column
		'''
		if colname is None:
			return self.__sterrValues
		if not self.isDataProcessed():
			raise ValueError("Data was not processed")
		col = self.getColumns().index(colname)
		return self.__sterrValues[:,col]

	def process(self):
		'''
		Processes the data
		'''
		centers = self.getBinCenters()
		self.__columns = self.getSource().getColumns()
		L = len(self.__columns)
		self.__counts = np.zeros(centers.size)
		self.__values = dict()
		self.__meanValues = np.zeros((centers.size, L))
		self.__stdValues = np.zeros((centers.size, L))
		self.__sterrValues = np.zeros((centers.size, L))
		for i in range(centers.size):
			leftBorder = self.getBinEdges()[i]
			rightBorder = self.getBinEdges()[i+1]
			hdata = FilteredMapTable.createFromData(self.getSource())
			hdata.setFiltrationValue(self.getBinningValue())
			hdata.setRange((leftBorder, rightBorder))
			hdata.process()
			self.__counts[i] = hdata.getRows()
			for col in self.__columns:
				iCol = self.__columns.index(col)
				if not self.__values.has_key(col):
					self.__values[col] = list()
				self.__values[col].append(hdata.getTable().copy()[:,iCol])
				self.__meanValues[i][iCol] = hdata.getAverage(col)
				self.__stdValues[i][iCol] = hdata.getStd(col)
				self.__sterrValues[i][iCol] = hdata.getStErr(col)
		self._setDataProcessed(True)

	def getMeanValuesTxt(self):
		'''
		Returns mean values as a text
		'''
		return self._getValuesTxt(self.__meanValues)

	def getStdValuesTxt(self):
		'''
		Returns standard deviations as a text
		'''
		return self._getValuesTxt(self.__stdValues)

	def getSterrValuesTxt(self):
		'''
		Return standard errors as a text
		'''
		return self._getValuesTxt(self.__sterrValues)

	def _getValuesTxt(self, matrix):
		S = "\t"
		for col in self.getColumns():
			S += "{0}\t".format(col)
		S += "\n\t"
		for col in self.getColumns():
			S += "=====\t"
		S += "\n"
		centers = self.getBinCenters()
		edges = self.getBinEdges()
		for i in range(centers.size):
			S += "{0}<=val<={1}\t".format(edges[i], edges[i+1])
			for j in range(len(self.getColumns())):
				S += "{0:.2f}\t".format(matrix[i,j])
			S += "\n"
		return S

	def getParameters(self):
		d = dict(value = self.getBinningValue())
		for i in range(self.getBinEdges().size):
			d["edge_{0}".format(i)] = str(self.getBinEdges()[i])
		return d

	def setParameters(self, d):
		self.setBinningValue(d['value'])
		edges = np.zeros(len(d)-1)
		for key, value in d.iteritems():
			if key.startswith("edge_"):
				idx = int(key[5:])
				edges[idx] = float(value)
		self.setBinEdges(edges)

	def getDataManagerClass(self):
		'''
		Returns a data manager class
		'''
		return Hist1DDataManager

	def write(self):
		self.getDataManager().write(self)

	def _readString(self, X):
		y =  X.tostring().replace('\x00', '')
		return y

	def read(self):
		d = self.getDataManager().read()
		self._setSpecimen(self._readString(d['cat']))
		self.getDataManager()._setSpecimen(self._readString(d['cat']))
		self._setExperiment(self._readString(d['experiment']))
		self.getDataManager()._setExperiment(self._readString(d['experiment']))
		self._setCondition(self._readString(d['condition']))
		self.getDataManager()._setCondition(self._readString(d['condition']))
		self.setBinEdges( d['bin_edges'][0] )
		self.setBinningValue( self._readString(d['binning_value']) )
		self.__columns = []
		self.__values = dict()
		for colname in d['columns'][0]:
			col = self._readString(colname)
			self.__columns.append(col)
			mcol = col.replace('-','_')
			cnt = d[mcol][0]
			self.__values[col] = list()
			for k in range(len(self.getBinCenters())):
				coldata = np.array(cnt[k][0])
				self.__values[col].append(coldata)
		self.__meanValues = d['mean_values']
		self.__stdValues = d['std_values']
		self.__sterrValues = d['sterr_values']
		self.__counts = d['counts']

	def plot(self, colname = None, ax = None):
		if ax is None:
			ax = plt.axes()
		centers = self.getBinCenters()
		bsize = self.getBinEdges()[1:] - self.getBinEdges()[:-1]
		if self.getBinningValue().find('orientation')!=-1:
			centers = centers*180/np.pi
			bsize = bsize*180/np.pi
		if colname is None:
			val = self.getPercentages()
			plt.ylabel('% of pixels')
		else:
			if colname.find('orientation')==-1 or colname=="orientation":
				val = self.getMeanValues(colname)
				if colname=="orientation":
					val = val*180/np.pi
			else:
				val = self.getStdValues(colname)*180/np.pi
			plt.ylabel(colname)
		h = ax.bar(centers, val, 0.9*bsize)
		plt.xlabel(self.getBinningValue())
		plt.title(self.getFullname())
		return h

	def plotGraph(self, colname, ax = None):
		if ax is None:
			ax = plt.axes()
		centers = self.getBinCenters()
		if self.getBinningValue().find('orientation')!=-1:
			centers = centers*180/np.pi
		if colname.find('orientation')==-1 or colname=="orientation":
			means = self.getMeanValues(colname)
			errors = self.getSterrValues(colname)
			h = ax.errorbar(centers, means, yerr=errors)
		else:
			std = self.getStdValues(colname)*180/np.pi
			h = ax.plot(centers, std)
		plt.xlabel(self.getBinningValue())
		plt.ylabel(colname)
		plt.title(self.getFullname())
		return h


import numpy as np
from fractions import gcd
from scipy.fftpack import fft

def optimalZeropad(x, fs, f):
    """
    Inputs:
    x (numpy array) = input signal of length M
    fs (float) = sampling frequency in Hz
    f (float) = frequency of the sinusoid in Hz
    Output:
    The function should return
    mX (numpy array) = The positive half of the DFT spectrum
    of the N point DFT after zero-padding input x
    appropriately (zero-padding length to be computed).
    mX is (N/2)+1 samples long
    """
    T = int(fs/f)
    M = len(x)
    zpn = T - (M % T)
    zp = np.zeros(zpn)
    x = np.append(x, zp)
    N = len(x)
    X = fft(x)
    mX = 20*np.log10(abs(X[:int(N/2)+1]))
    return mX

def genSine(A, f, phi, fs, t):
    """
    Inputs:
    A (float) = amplitude of the sinusoid
    f (float) = frequency of the sinusoid in Hz
    phi (float) = initial phase of the sinusoid in radians
    fs (float) = sampling frequency of the sinusoid in Hz
    t (float) = duration of the sinusoid (is second)
    Output:
    The function should return a numpy array
    x (numpy array) = The generated sinusoid (use np.cos())
    """
    T = 1/fs
    n = np.arange(0, t, T)
    x = A*np.cos(2*(np.pi)*f*n + phi)
    return x

fs = 1000
f = 100
M = 25
phi = 0
A = 1
t = 0.025
x = genSine(A,f, phi, fs, t)
output = optimalZeropad(x, fs, f)
print(output)


import boto3
from boto3.dynamodb.conditions import Key, Attr
from subactions import new_uuid

class Table():
    table = None
    dynamodb = boto3.resource('dynamodb', region_name='us-east-2')

    def __init__(self, table_name):
        self.table = self.dynamodb.Table(table_name)

    def get_item(self, UUID):
        return self.table.get_item(Key={'UUID': UUID})['Item']

    def put_item(self, item):
        clone = item

        for key, val in item.items():
            if val == set():
                del clone[key]

        return self.table.put_item(Item=clone)

    def delete_item(self, UUID):
        return self.table.delete_item(Key={'UUID': UUID})

    def query(self, index_name, key_condition_expression):
        return self.table.query(IndexName=index_name, KeyConditionExpression=key_condition_expression)['Items']

class AssiciatedTable(Table):
    association_table = None
    associated_table = None
    index_name = None
    key_name = None
    associated_index_name = None
    associated_key_name = None

    def __init__(self, table_name, association_table_name, associated_table_name, index_name, key_name, associated_index_name, associated_key_name):
        super().__init__(table_name)
        self.association_table = Table(association_table_name)
        self.associated_table = Table(associated_table_name)
        self.index_name = index_name
        self.key_name = key_name
        self.associated_index_name = associated_index_name
        self.associated_key_name = associated_key_name

    def __get_associations(self, uuid):
        return self.association_table.query(
            self.index_name,
            Key(self.key_name).eq(uuid),
        )

    def __associate(self, item, alias="associatedItems", only_associate_uuid=False):
        associations = self.__get_associations(item['UUID'])

        if only_associate_uuid:
            item[alias] = []
            for association in associations:
                associated_uuid = association[self.associated_key_name]
                item[alias].append(associated_uuid)
        else:
            item[alias] = {}
            for association in associations:
                associated_uuid = association[self.associated_key_name]
                associated_item = self.associated_table.get_item(
                    associated_uuid)
                item[alias][associated_uuid] = associated_item
        return item

    def query(self, index_name, key_condition_expression, with_associated_items=False, alias="associatedItem", only_associate_uuid=True):
        items = super().query(index_name, key_condition_expression)

        if with_associated_items:
            for item in items:
                item = self.__associate(item, alias, only_associate_uuid)
        return items

    def get_item(self, UUID, with_associated_items, alias, only_associate_uuid):
        item = super().get_item(UUID)

        if with_associated_items:
            item = self.__associate(item, alias, only_associate_uuid)

        return item

    # the associated_items must already exist
    def put_item(self, item, associated_item_uuids=None):
        if ('UUID' not in item) or item['UUID'] is -1:
            item['UUID'] = new_uuid()

        super().put_item(item)

        if associated_item_uuids:
            for association in self.__get_associations(item['UUID']):
                self.association_table.delete_item(association['UUID'])

            # then call this
            for associated_item_uuid in associated_item_uuids:
                self.association_table.put_item({
                    'UUID': new_uuid(),
                    self.key_name: item['UUID'],
                    self.associated_key_name: associated_item_uuid
                })

        return item['UUID']

    def delete_item(self, UUID):
        associations = self.__get_associations(UUID)
        super().delete_item(UUID)
        for association in associations:
            self.association_table.delete_item(association['UUID'])

        return None


# """Tools for constructing quantum circuits."""
import json
import warnings
from math import pi

import cirq
import numpy as np
import pyquil
import qiskit
from pyquil import Program
from pyquil.gates import MEASURE, I
from pyquil.quilatom import quil_cos, quil_sin

from ..utils import SCHEMA_VERSION
from ._gate import Gate
from ._gateset import COMMON_GATES, UNIQUE_GATES
from ._qubit import Qubit

class Circuit(object):
    """Base class for quantum circuits.

    Attributes:
        name: string
            Name of the Circuit object. By default this is called 'Unnamed'.
        gates: list[Gate]
            The gate sequence of the circuit. Implemented as a list of core.gate.Gate
            objects.
        qubits: list[Qubit]
            The set of qubits that the circuit acts on. Implemented as a list of
            core.qubit.Qubit objects.
        info: dictionary
            Additional information related to the circuit. For example if the circuit is
            converted from another package, information related to the native
            specification of the circuit in that package is recorded here.
    """

    def __init__(self, input_object=None, name="Unnamed"):
        """Initialize a circuit. Most likely the circuit is generated by converting a
        circuit object in other packages to core.circuit.Circuit object.

        Args:
            input_object: pyquil.Program, cirq.Circuit, qiskit.QuantumCircuit
                A generic circuit object that may be created from one of the various
                packages currently supported by Zap OS.
        """

        self.name = name  # name of the circuit (by default a random Hash string)
        self.gates = []  # list of gates (see gate.py for Gate class def)
        self.qubits = []  # list of qubits (see qubit.py for Qubit class def)
        self.info = {
            "label": None  # the name of the native package that generates the circuit
            # e.g. 'pyquil', 'cirq', 'qiskit' etc. The purpose is to
            # provide hints about what unique functionalities of
            # the package one might be able to take advantage of.
        }

        if isinstance(input_object, pyquil.Program):
            self.from_pyquil(input_object)
        elif isinstance(input_object, pyquil.quilbase.Gate):
            converted_input = pyquil.Program(input_object)
            self.from_pyquil(converted_input)
        elif isinstance(input_object, cirq.Circuit):
            self.from_cirq(input_object)
        elif isinstance(input_object, qiskit.QuantumCircuit):
            self.from_qiskit(input_object)
        elif input_object is None:
            pass
        else:
            raise (
                TypeError(
                    "Incorrect type of input object: {0}".format(type(input_object))
                )
            )

    @property
    def n_multiqubit_gates(self):
        """The number of multiqubit gates in the circuit."""

        n_mq_gates = 0
        for gate in self.gates:
            if len(gate.qubits) > 1:
                n_mq_gates += 1

        return n_mq_gates

    @property
    def symbolic_params(self):
        """
        Returns a set of symbolic parameters used in the circuit in the chronological
        order.

        Returns:
            list: list of all the sympy symbols used as params of gates in the circuit.
        """
        symbolic_params = []
        for gate in self.gates:
            symbolic_params_per_gate = gate.symbolic_params
            for param in symbolic_params_per_gate:
                if param not in symbolic_params:
                    symbolic_params.append(param)

        return symbolic_params

    def __eq__(self, anotherCircuit):
        """Comparison between two Circuit objects."""
        if self.name != anotherCircuit.name:
            return False
        if len(self.qubits) != len(anotherCircuit.qubits):
            return False
        for i in range(len(self.qubits)):
            if str(self.qubits[i]) != str(anotherCircuit.qubits[i]):
                return False

        if len(self.gates) != len(anotherCircuit.gates):
            return False
        for i in range(len(self.gates)):
            if self.gates[i] != anotherCircuit.gates[i]:
                return False

        return True

    def __add__(self, other_circuit):
        """Add two circuits."""

        qubit_indices = set(
            [qubit.index for qubit in self.qubits]
            + [qubit.index for qubit in other_circuit.qubits]
        )

        new_circuit = Circuit()

        for qubit_index in qubit_indices:
            new_circuit.qubits.append(Qubit(qubit_index))

        new_circuit.gates = self.gates + other_circuit.gates

        return new_circuit

    def get_qubits(self):
        """Returns a list of qubit indices (ints)."""

        return [q.index for q in self.qubits]

    def evaluate(self, symbols_map):
        """
        Returns a copy of a circuit with specified symbolic parameters evaluated to
        provided values.

        Args:
            symbols_map list(tuple(sympy.Basic, number)): List containing symbols and
            values that they should take.
        """
        new_circuit = type(self)()
        new_circuit.name = self.name
        new_circuit.qubits = self.qubits
        new_circuit.info = self.info
        gates = []

        all_symbols_in_map = set([item[0] for item in symbols_map])
        if len(all_symbols_in_map - set(self.symbolic_params)) > 0:
            warnings.warn(
                """
                Trying to evaluate circuit with symbols not existing in the circuit:
                Symbols in circuit: {0}
                Symbols in the map: {1}
                """.format(
                    self.symbolic_params, all_symbols_in_map
                ),
                Warning,
            )

        for gate in self.gates:
            gates.append(gate.evaluate(symbols_map))

        new_circuit.gates = gates
        return new_circuit

    def to_pyquil(self):
        """Converts the circuit to a pyquil Program object."""

        output = Program()
        if self.gates is not None:
            for gate in self.gates:
                output = add_gate_to_pyquil_program(output, gate)
        return output

    def to_cirq(self, cirq_qubits=None):
        """Converts the circuit to a cirq Circuit object.
        NOTE: Here we always assume that the resulting circuit acts on a linear chain of
        qubits.

        Args:
            cirq_qubits: list[cirq.LineQubit]
                (optional) A list of cirq.LineQubit objects.
        """

        qubits = []
        if cirq_qubits is None:
            if self.qubits is not None:
                if self.info["label"] == "cirq":
                    for q in self.qubits:
                        qkey = q.info["QubitKey"]
                        if q.info["QubitType"] == "GridQubit":
                            qubits.append(cirq.GridQubit(qkey[0], qkey[1]))
                        if q.info["QubitType"] == "LineQubit":
                            qubits.append(cirq.LineQubit(qkey))
                else:
                    qubits = [cirq.LineQubit(i) for i in self.get_qubits()]
        else:
            if len(cirq_qubits) < len(self.qubits):
                raise Exception(
                    f"Input qubit register size is {len(cirq_qubits)}, which is not "
                    "enough to represent this Circuit object that acts on "
                    f"{len(self.qubits)} qubits"
                )
            qubits = cirq_qubits

        if self.gates is not None:
            gates = [g.to_cirq(cirq_qubits) for g in self.gates]
        else:
            gates = []

        cirq_circuit = cirq.Circuit()
        cirq_circuit.append(gates, strategy=cirq.circuits.InsertStrategy.EARLIEST)
        return cirq_circuit

    def to_qiskit(self):
        """Converts the circuit to a qiskit QuantumCircuit object."""
        qiskit_circuit = qiskit.QuantumCircuit()  # New qiskit circuit object
        qreg = None
        creg = None

        if (
            self.qubits is not None and self.qubits != []
        ):  # If there are qubits in the circuit, add them to the new qiskit circuit
            max_qindex = max([q.index for q in self.qubits])
            qreg = qiskit.QuantumRegister(max_qindex + 1, "q")
            creg = qiskit.ClassicalRegister(max_qindex + 1, "c")
            qiskit_circuit.add_register(qreg)
            qiskit_circuit.add_register(creg)

        if self.gates is not None:
            for gate in self.gates:
                qiskit_gate_data = gate.to_qiskit(
                    qreg, creg
                )  # provide the gate conversion with the associated QuantumRegister

                # total number of entries in the list (which is 3x the number of
                # elementary gates)
                N = len(
                    qiskit_gate_data
                )

                if N % 3 != 0:
                    raise ValueError(
                        f"The number of entries in qiskit_gate_data is {N} which is not"
                        " a multiple of 3"
                    )
                for index in np.linspace(0, N - 3, N // 3):
                    qiskit_circuit.append(
                        qiskit_gate_data[int(index)],
                        qargs=qiskit_gate_data[int(index) + 1],
                        cargs=qiskit_gate_data[int(index) + 2],
                    )

        return qiskit_circuit

    def to_dict(self, serialize_gate_params=True):
        """Creates a dictionary representing a circuit.

        Args:
            serialize_gate_params(bool): if true, it will change gate params from sympy
            to strings (if applicable).

        Returns:
            dictionary (dict): the dictionary
        """

        if self.gates is not None:
            gates_entry = [
                gate.to_dict(serialize_params=serialize_gate_params)
                for gate in self.gates
            ]
        else:
            gates_entry = None

        if self.qubits is not None:
            qubits_entry = [qubit.to_dict() for qubit in self.qubits]
        else:
            qubits_entry = None

        dictionary = {
            "schema": SCHEMA_VERSION + "-circuit",
            "name": self.name,
            "gates": gates_entry,
            "qubits": qubits_entry,
            "info": self.info,
        }

        return dictionary

    def to_unitary(self):
        """Creates a unitary matrix representing the circuit.

        Returns:
            An array representing the unitary matrix.
        """

        return self.to_cirq()._unitary_()

    def to_text_diagram(self, transpose=False):
        """Gets a text diagram representing the circuit.

        transpose (bool): if true, arrange qubit wires vertically instead of
            horizontally.

        Returns:
            str: a string containing the text diagram
        """

        return self.to_cirq().to_text_diagram(transpose=transpose)

    def to_quil(self):
        """Gets the quil program representing the circuit.
        Returns:
            str: a string containing the quil program
        """
        return self.to_pyquil().out()

    def to_qpic(self):
        """Generates a string that can be used by qpic to build a picture of the
        circuit.

        Returns:
            str: a qpic string
        """

        qpic_string = ""

        for qubit in sorted(self.qubits, key=lambda q: q.index):
            qpic_string += "w{} W {}\n".format(qubit.index, qubit.index)

        for gate in self.gates:
            qpic_string += gate.to_qpic() + "\n"

        return qpic_string

    def __str__(self):
        """Get a string representation of the circuit.
        Returns:
            str: a string representation of the circuit
        """
        return self.to_text_diagram()

    @classmethod
    def from_dict(cls, dictionary):
        """Loads information of the circuit from a dictionary. This corresponds to the
        serialization routines to_dict for Circuit, Gate and Qubit.

        Args:
            dictionary (dict): the dictionary

        Returns:
            A core.circuit.Circuit object
        """

        output = cls(name=dictionary["name"])
        if dictionary["gates"] is not None:
            output.gates = [Gate.from_dict(gate) for gate in dictionary["gates"]]
        else:
            output.gates = None

        if dictionary["qubits"] is not None:
            output.qubits = [Qubit.from_dict(qubit) for qubit in dictionary["qubits"]]
        else:
            output.qubits = None
        output.info = dictionary["info"]
        return output

    def from_pyquil(self, pyquil_circuit):
        """Converts a pyquil Program object to a core.Circuit object.

        Args:
            pyquil_circuit: Program object(pyquil)
            name: string
                Name of the converted core.Circuit object.

        """

        self.info["label"] = "pyquil"

        _gatelist = []
        _qubits = []

        if len(pyquil_circuit) == 0:
            return

        _pyquil_qubits = []  # list of currently found *pyquil* qubits
        for gate in pyquil_circuit:

            _gatequbits = []
            for qubit in gate.qubits:

                def qubit_in_list(
                    qubit, qubitlist
                ):  # check if a pyquil qubit is in a list of pyquil qubits
                    output = False
                    out_index = []
                    for q in qubitlist:
                        if qubit.index == q.index:
                            output = True
                            out_index = q.index
                            break
                    return output, out_index

                _flag, _index = qubit_in_list(qubit, _pyquil_qubits)
                if _flag is False:
                    _pyquil_qubits.append(qubit)
                    _new_Qubit = Qubit.from_pyquil(qubit)
                    _qubits.append(_new_Qubit)
                    _gatequbits.append(_new_Qubit)
                else:
                    for q in _qubits:
                        if q.index == _index:
                            _old_Qubit = q
                            break
                    _gatequbits.append(_old_Qubit)

            _gatelist.append(Gate.from_pyquil(gate, _gatequbits))

        self.gates = _gatelist
        self.qubits = _qubits

    def from_cirq(self, cirq_circuit):
        """Convert from a cirq Circuit object to a core.Circuit object.

        Args:
            cirq_circuit: cirq Cirquit object.
                See the following: https://github.com/quantumlib/Cirq

        """
        self.info["label"] = "cirq"

        _gatelist = []
        _qubits = []

        if (
            len(cirq_circuit) == 0
            or sum([len(m.operations) for m in cirq_circuit]) == 0
        ):
            return

        _cirq_qubits = []  # list of currently found *cirq* qubits
        for moment in cirq_circuit:
            for op in moment.operations:
                _gatequbits = []
                for qubit in op.qubits:

                    def qubit_in_list(
                        qubit, qubitlist
                    ):  # check if a cirq qubit is in a list of cirq qubits
                        # if yes return the index
                        output = False
                        out_index = []
                        for q in qubitlist:
                            if isinstance(qubit, cirq.GridQubit) and isinstance(
                                q, cirq.GridQubit
                            ):
                                if qubit.row == q.row and qubit.col == q.col:
                                    output = True
                                    out_index = (q.row, q.col)
                                    break
                            elif isinstance(qubit, cirq.LineQubit) and isinstance(
                                qubit, cirq.LineQubit
                            ):
                                if qubit.x == q.x:
                                    output = True
                                    out_index = q.x
                                    break
                            else:
                                raise TypeError(
                                    "(Cirq) Qubit and Qubit list elements not of the "
                                    "same kind."
                                )
                        return output, out_index

                    _flag, _index = qubit_in_list(qubit, _cirq_qubits)
                    if _flag is False:  # if the qubit is not seen before
                        _cirq_qubits.append(
                            qubit
                        )  # add the cirq qubit to the list of cirq qubits seen
                        _new_Qubit = Qubit.from_cirq(
                            qubit, qubit.x
                        )  # generate a new qubit
                        _qubits.append(_new_Qubit)
                        _gatequbits.append(_new_Qubit)
                    else:  # if the qubit is already seen before
                        for (
                            q
                        ) in (
                            _qubits
                        ):  # search for the old Qubit object in the _qubits list
                            if q.info["QubitKey"] == _index:
                                _old_Qubit = q
                                break
                        _gatequbits.append(_old_Qubit)
                _gatelist.append(Gate.from_cirq(op, _gatequbits))

        self.gates = _gatelist
        self.qubits = _qubits

    def from_qiskit(self, qiskit_circuit):
        """Convert from a qiskit QuantumCircuit object to a core.circuit.Circuit object.

        Args:
            qiskit_circuit: qiskit QuantumCircuit object.

        """

        self.name = qiskit_circuit.name
        self.info["label"] = "qiskit"

        _gatelist = []  # list of gates for the output Circuit object
        _qubits = []  # list of qubits for the output Circuit object

        if len(qiskit_circuit.data) == 0:
            return

        _qiskit_qubits = []  # list of qiskit qubits in the circuit object
        for gate_data in qiskit_circuit.data:
            _gatequbits = []
            for qubit in gate_data[1]:

                def qubit_in_list(
                    qubit, qubitlist
                ):  # check if a qiskit qubit is in a list of qiskit qubit
                    output = False
                    for q in qubitlist:
                        if qubit == q:
                            output = True
                            break
                    return output

                if (
                    qubit_in_list(qubit, _qiskit_qubits) == 0
                ):  # if the qubit is not seen before
                    _qiskit_qubits.append(
                        qubit
                    )  # add the qiskit qubit to the list of seen qiskit qubits
                    _new_Qubit = Qubit.from_qiskit(
                        qubit, qubit.index
                    )  # generate a new Qubit object
                    _qubits.append(
                        _new_Qubit
                    )  # add to the list of Qubit objects for the output Circuit object
                    _gatequbits.append(
                        _new_Qubit
                    )  # add to the list of Qubit objects that the gate acts on
                else:  # if the qubit is already seen before
                    for (
                        q
                    ) in _qubits:  # search for the old Qubit object in the _qubits list
                        if q.info["num"] == qubit.index:
                            _old_Qubit = q
                            break
                    _gatequbits.append(_old_Qubit)

            zap_gate = Gate.from_qiskit(gate_data[0], _gatequbits)
            if zap_gate is not None:
                _gatelist.append(zap_gate)

        self.gates = _gatelist
        self.qubits = _qubits

def save_circuit(circuit, file_or_path):
    """Saves a circuit object to a file.

    Args:
        circuit: the circuit to be saved
        file_or_path: the name of the file
    """
    payload = circuit.to_dict(serialize_gate_params=True)
    try:
        json.dump(payload, file_or_path)
    except AttributeError:
        with open(file_or_path, "w") as f:
            json.dump(payload, f)

def load_circuit(file):
    """Loads a circuit from a file.

    Args:
        file (str or file-like object): the name of the file, or a file-like object.

    Returns:
        circuit (core.Circuit): the circuit
    """

    if isinstance(file, str):
        with open(file, "r") as f:
            data = json.load(f)
    else:
        data = json.load(file)

    return Circuit.from_dict(data)

def save_circuit_set(circuit_set, filename):
    """Save a circuit set to a file.

    Args:
        circuit_set (list): a list of core.Circuit objects
        file (str or file-like object): the name of the file, or a file-like object
    """
    dictionary = {}
    dictionary["schema"] = SCHEMA_VERSION + "-circuit_set"
    dictionary["circuits"] = []
    for circuit in circuit_set:
        dictionary["circuits"].append(circuit.to_dict(serialize_gate_params=True))
    with open(filename, "w") as f:
        f.write(json.dumps(dictionary, indent=2))

def load_circuit_set(file):
    """Load a set of circuits from a file.

    Args:
        file (str or file-like object): the name of the file, or a file-like object.

    Returns:
        circuit_set (list): a list of core.Circuit objects
    """
    if isinstance(file, str):
        with open(file, "r") as f:
            data = json.load(f)
    else:
        data = json.load(file)

    circuit_set = []
    for circuit_dict in data["circuits"]:
        circuit_set.append(Circuit.from_dict(circuit_dict))
    return circuit_set

def pyquil2cirq(qprog):
    """Convert a pyquil Program to a cirq Circuit.

    Currently supports only common single- and two-qubit gates.

    Args:
        qprog (pyquil.quil.Program): the program to be converted.

    Returns:
        circuit (cirq.Cirquit): the converted circuit"""

    # A map between gate names used by pyquil and cirq gate objects
    op_map = {
        "X": cirq.X,
        "Y": cirq.Y,
        "Z": cirq.Z,
        "T": cirq.T,
        "H": cirq.H,
        "S": cirq.S,
        "RX": cirq.XPowGate,
        "RY": cirq.YPowGate,
        "RZ": cirq.ZPowGate,
        "CNOT": cirq.CNOT,
        "SWAP": cirq.SWAP,
        "CZ": cirq.CZ,
        "CPHASE": cirq.ops.common_gates.CZPowGate,
    }

    # Create the qubits. The row of each grid qubit is equal to the index
    # of the corresponding pyquil qubit.
    qubits = [cirq.GridQubit(i, 0) for i in qprog.get_qubits()]

    # A map between the row of the qubit and the index in the qubits array
    qubit_map = {}
    for i in range(len(qubits)):
        qubit_map[qubits[i].row] = i

    circuit = cirq.Circuit()

    for gate in qprog:
        if not op_map.get(gate.name):
            raise ValueError("Gate {} not yet supported".format(gate.name))

        # Find the cirq qubits that this gate acts on
        target_qubits = [qubits[qubit_map[q.index]] for q in gate.qubits]

        # Create the cirq gate
        if len(gate.params) == 0:
            cirq_gate = op_map[gate.name](*target_qubits)
        elif len(gate.params) == 1:
            cirq_gate = op_map[gate.name](exponent=gate.params[0] / np.pi)(
                *target_qubits
            )
        else:
            raise ValueError(
                "Gates with more than one parameter not yet supported: {}".format(gate)
            )

        # Append the gate to the circuit
        circuit.append(cirq_gate, strategy=cirq.circuits.InsertStrategy.EARLIEST)

    return circuit

def cirq2pyquil(circuit):
    """Convert a cirq Circuit to a pyquil Program.

    Currently supports only common single- and two-qubit gates.

    Args:
        circuit (cirq.Cirquit): the converted circuit.

    Returns:
        qprog (pyquil.quil.Program): the program to be converted."""

    # A map between cirq gate string representations and pyquil gate classes
    op_repr_map = {
        "cirq.X": pyquil.gates.X,
        "cirq.Y": pyquil.gates.Y,
        "cirq.Z": pyquil.gates.Z,
        "cirq.T": pyquil.gates.T,
        "cirq.H": pyquil.gates.H,
        "cirq.S": pyquil.gates.S,
        "cirq.CNOT": pyquil.gates.CNOT,
        "cirq.SWAP": pyquil.gates.SWAP,
        "cirq.CZ": pyquil.gates.CZ,
    }

    # A map between cirq gate classes and pyquil gate classes. Perhaps better to parse
    # repr?
    op_type_map = {
        cirq.ops.common_gates.XPowGate: pyquil.gates.RX,
        cirq.ops.common_gates.YPowGate: pyquil.gates.RY,
        cirq.ops.common_gates.ZPowGate: pyquil.gates.RZ,
        cirq.ops.common_gates.CZPowGate: pyquil.gates.CPHASE,
    }

    # Create a map from row/column tuples to linear qubit index
    qubit_map = {}
    qubit_count = 0
    qubit = next(iter(circuit.all_qubits()))  # Grab a random qubit
    if isinstance(qubit, cirq.GridQubit):
        def qubit_key(q):
            return (q.row, q.col)
    elif isinstance(qubit, cirq.LineQubit):
        def qubit_key(q):
            return q.x
    else:
        raise ValueError("Qubit type {} not yet supported".format(type(qubit)))
    for qubit in sorted(circuit.all_qubits(), key=qubit_key):
        qubit_map[qubit_key(qubit)] = qubit_count
        qubit_count += 1

    # Create the program
    qprog = pyquil.quil.Program()

    def add_to_program(op):
        """Add a cirq op to the pyquil program qprog."""

        # Find the linear indices of the qubits acted on by this operation
        qubits = [qubit_map[qubit_key(q)] for q in op.qubits]

        # First check if the string representation matches known gates
        if op_repr_map.get(repr(op.gate)):
            qprog.inst(op_repr_map[repr(op.gate)](*qubits))

        # Next check if the type of the gate object matches known gates
        elif op_type_map.get(type(op.gate)):
            rads = op.gate.exponent * np.pi
            pyquil_gate = op_type_map[type(op.gate)]
            qprog.inst(pyquil_gate(rads, *qubits))

        # Decompose if PhasedXPowGate or HPowGate
        elif isinstance(op.gate, cirq.PhasedXPowGate) or isinstance(
            op.gate, cirq.HPowGate
        ):
            ops = cirq.decompose(op)
            for op in ops:
                add_to_program(op)

        elif isinstance(op.gate, cirq.XXPowGate):
            q1, q2 = op.qubits
            ops = [
                cirq.H(q1),
                cirq.H(q2),
                cirq.CNOT(q1, q2),
                cirq.rz(op.gate.exponent * pi)(q2),
                cirq.CNOT(q1, q2),
                cirq.H(q1),
                cirq.H(q2),
            ]
            for op in ops:
                add_to_program(op)

        elif isinstance(op.gate, cirq.YYPowGate):
            q1, q2 = op.qubits
            ops = [
                cirq.Z(q1) ** 0.5,
                cirq.Z(q2) ** 0.5,
                cirq.H(q1),
                cirq.H(q2),
                cirq.CNOT(q1, q2),
                cirq.rz(op.gate.exponent * pi)(q2),
                cirq.CNOT(q1, q2),
                cirq.H(q1),
                cirq.H(q2),
                cirq.Z(q1) ** -0.5,
                cirq.Z(q2) ** -0.5,
            ]
            for op in ops:
                add_to_program(op)

        elif isinstance(op.gate, cirq.ZZPowGate):
            q1, q2 = op.qubits
            ops = [
                cirq.CNOT(q1, q2),
                cirq.rz(op.gate.exponent * pi)(q2),
                cirq.CNOT(q1, q2),
            ]
            for op in ops:
                add_to_program(op)

        else:
            raise ValueError("Gate {} not yet supported".format(op.gate))

    for moment in circuit:
        for op in moment.operations:
            add_to_program(op)

    return qprog

def add_gate_to_pyquil_program(pyquil_program, gate):
    """Add the definition of a gate to a pyquil Program object if the gate is
    not currently defined.

    Args:
        pyquil_program: pyquil.Program
            The input Program object to which the gate is going to be added.
        gate: Gate (core.circuit)
            The Gate object describing the gate to be added.

    Returns:
        A new pyquil.Program object with the definition of the new gate being added.
    """

    if gate.name in COMMON_GATES:  # if a gate is already included in pyquil
        return pyquil_program + gate.to_pyquil()  # do nothing
    elif gate.name in UNIQUE_GATES:  # if a gate is unique to a specific package
        if gate.name == "ZXZ":
            beta = pyquil.quilatom.Parameter("beta")
            gamma = pyquil.quilatom.Parameter("gamma")
            zxz_unitary = np.array(
                [
                    [
                        quil_cos(gamma / 2),
                        -quil_sin(beta) * quil_sin(gamma / 2)
                        - 1j * quil_cos(beta) * quil_sin(gamma / 2),
                    ],
                    [
                        quil_sin(beta) * quil_sin(gamma / 2)
                        - 1j * quil_cos(beta) * quil_sin(gamma / 2),
                        quil_cos(gamma / 2),
                    ],
                ]
            )
            zxz_def = pyquil.quilbase.DefGate("ZXZ", zxz_unitary, [beta, gamma])
            ZXZ = zxz_def.get_constructor()
            return (
                pyquil_program
                + zxz_def
                + ZXZ(gate.params[0], gate.params[1])(gate.qubits[0].index)
            )
        if gate.name == "RH":
            beta = pyquil.quilatom.Parameter("beta")
            phase_factor = quil_cos(beta / 2) + 1j * quil_sin(beta / 2)
            elem00 = quil_cos(beta / 2) - 1j * 1 / np.sqrt(2) * quil_sin(beta / 2)
            elem01 = -1j * 1 / np.sqrt(2) * quil_sin(beta / 2)
            elem10 = -1j * 1 / np.sqrt(2) * quil_sin(beta / 2)
            elem11 = quil_cos(beta / 2) + 1j * 1 / np.sqrt(2) * quil_sin(beta / 2)
            rh_unitary = np.array(
                [
                    [phase_factor * elem00, phase_factor * elem01],
                    [phase_factor * elem10, phase_factor * elem11],
                ]
            )
            rh_def = pyquil.quilbase.DefGate("RH", rh_unitary, [beta])
            RH = rh_def.get_constructor()
            return pyquil_program + rh_def + RH(gate.params[0])(gate.qubits[0].index)
        if gate.name == "XX":
            # Reference for XX implementation in cirq:
            # https://github.com/quantumlib/Cirq/blob/a61e51b53612735e93b3bb8a7605030c499cd6c7/cirq/ops/parity_gates.py#L30
            # Reference for XX implementation in qiskit:
            # https://qiskit.org/documentation/stubs/qiskit.circuit.library.RXXGate.html
            beta = pyquil.quilatom.Parameter("beta")
            elem_cos = quil_cos(beta)
            elem_sin = -1j * quil_sin(beta)
            xx_unitary = np.array(
                [
                    [elem_cos, 0, 0, elem_sin],
                    [0, elem_cos, elem_sin, 0],
                    [0, elem_sin, elem_cos, 0],
                    [elem_sin, 0, 0, elem_cos],
                ]
            )
            xx_def = pyquil.quilbase.DefGate("XX", xx_unitary, [beta])
            XX = xx_def.get_constructor()
            return (
                pyquil_program
                + xx_def
                + XX(gate.params[0])(gate.qubits[0].index, gate.qubits[1].index)
            )
        if gate.name == "YY":
            # Reference for YY implementation in cirq:
            # https://github.com/quantumlib/Cirq/blob/a61e51b53612735e93b3bb8a7605030c499cd6c7/cirq/ops/parity_gates.py#L142
            # Reference for YY implementation in qiskit:
            # https://qiskit.org/documentation/stubs/qiskit.circuit.library.RYYGate.html
            beta = pyquil.quilatom.Parameter("beta")
            elem_cos = quil_cos(beta)
            elem_sin = 1j * quil_sin(beta)
            yy_unitary = np.array(
                [
                    [elem_cos, 0, 0, elem_sin],
                    [0, elem_cos, -elem_sin, 0],
                    [0, -elem_sin, elem_cos, 0],
                    [elem_sin, 0, 0, elem_cos],
                ]
            )
            yy_def = pyquil.quilbase.DefGate("YY", yy_unitary, [beta])
            YY = yy_def.get_constructor()
            return (
                pyquil_program
                + yy_def
                + YY(gate.params[0])(gate.qubits[0].index, gate.qubits[1].index)
            )
        if gate.name == "ZZ":
            # Reference for ZZ implementation in cirq:
            # https://github.com/quantumlib/Cirq/blob/a61e51b53612735e93b3bb8a7605030c499cd6c7/cirq/ops/parity_gates.py#L254
            # Reference for ZZ implementation in qiskit:
            # https://qiskit.org/documentation/stubs/qiskit.circuit.library.RYYGate.html
            beta = pyquil.quilatom.Parameter("beta")
            elem_cos = quil_cos(beta)
            elem_sin = 1j * quil_sin(beta)
            zz_unitary = np.array(
                [
                    [elem_cos - elem_sin, 0, 0, 0],
                    [0, elem_cos + elem_sin, 0, 0],
                    [0, 0, elem_cos + elem_sin, 0],
                    [0, 0, 0, elem_cos - elem_sin],
                ]
            )
            zz_def = pyquil.quilbase.DefGate("ZZ", zz_unitary, [beta])
            ZZ = zz_def.get_constructor()
            return (
                pyquil_program
                + zz_def
                + ZZ(gate.params[0])(gate.qubits[0].index, gate.qubits[1].index)
            )
        if gate.name == "XY":
            return pyquil_program + gate.to_pyquil()  # do nothing

        if gate.name == "U1ex":  # IBM U1ex gate (arXiv:1805.04340v1)
            alpha = pyquil.quilatom.Parameter("alpha")
            beta = pyquil.quilatom.Parameter("beta")
            elem_cos = quil_cos(beta)
            elem_sin = 1j * quil_sin(beta)
            unitary = [[1, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 1]]
            unitary[1][1] = quil_cos(alpha)
            unitary[2][2] = -quil_cos(alpha)
            unitary[2][1] = (quil_cos(beta) - 1j * quil_sin(beta)) * quil_sin(alpha)
            unitary[1][2] = (quil_cos(beta) + 1j * quil_sin(beta)) * quil_sin(alpha)
            u1ex_def = pyquil.quilbase.DefGate("U1ex", np.array(unitary), [alpha, beta])
            U1ex = u1ex_def.get_constructor()
            output_program = pyquil_program + U1ex(gate.params[0], gate.params[1])(
                gate.qubits[0].index, gate.qubits[1].index
            )
            gate_already_defined = False
            for gate_definition in pyquil_program.defined_gates:
                if gate_definition.name == "U1ex":
                    gate_already_defined = True
                    break
            if not gate_already_defined:
                output_program = output_program + u1ex_def
            return output_program
        if gate.name == "U2ex":  # IBM U2ex gate (arXiv:1805.04340v1)
            alpha = pyquil.quilatom.Parameter("alpha")
            unitary = [[1, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 1]]
            unitary[1][1] = quil_cos(2 * alpha)
            unitary[2][2] = quil_cos(2 * alpha)
            unitary[2][1] = -1j * quil_sin(2 * alpha)
            unitary[1][2] = -1j * quil_sin(2 * alpha)
            u2ex_def = pyquil.quilbase.DefGate("U2ex", np.array(unitary), [alpha])
            U2ex = u2ex_def.get_constructor()
            output_program = pyquil_program + U2ex(gate.params[0])(
                gate.qubits[0].index, gate.qubits[1].index
            )
            gate_already_defined = False
            for gate_definition in pyquil_program.defined_gates:
                if gate_definition.name == "U2ex":
                    gate_already_defined = True
                    break
            if not gate_already_defined:
                output_program = output_program + u1ex_def
            return output_program
        if gate.name == "MEASURE":
            reg_name = "r" + str(gate.qubits[0].index)

            ro = pyquil_program.declare(reg_name, "BIT", 1)
            return pyquil_program + MEASURE(gate.qubits[0].index, ro[0])
        if gate.name == "BARRIER":
            return pyquil_program

def add_ancilla_register_to_circuit(circuit, n_qubits_ancilla_register):
    """Add a register of ancilla qubits (qubit + identity gate) to an existing circuit.

    Args:
        circuit (core.Circuit): circuit to be extended
        n_qubits_ancilla_register (int): number of ancilla qubits to add
    Returns:
        core.Circuit: extended circuit

    """
    extended_circuit = Circuit()
    n_qubits = len(circuit.qubits)
    pyquil_circuit = circuit.to_pyquil()
    for i in range(n_qubits_ancilla_register):
        pyquil_circuit += I(n_qubits + i)
    extended_circuit.from_pyquil(pyquil_circuit)
    return extended_circuit


#!/usr/bin/python3
""" This module changes the name of a State object
from the database hbtn_0e_6_usa"""
import sys
from model_state import Base, State
from sqlalchemy import (create_engine)
from sqlalchemy.orm import sessionmaker

if __name__ == "__main__":
    engine = create_engine('mysql+mysqldb://{}:{}@localhost:3306/{}'
                           .format(sys.argv[1], sys.argv[2], sys.argv[3]))
    Base.metadata.create_all(engine)
    """Session object provides the entrypoint to acquire a Query object
    """
    Session = sessionmaker(bind=engine)

    session = Session()
    """query() takes one or more entities and returns a
    new Query object which will issue mapper queries within
    the context of this Session
    """
    state = session.query(State).filter(State.id == "2")\
                                .update({State.name: "New Mexico"})
    session.commit()
    session.close()