import sys
import os
import gc
import math
import numpy as np
import cv2
import glob
import imageio
from PIL import Image
from random import choice
from keras.utils import Sequence
import matplotlib.pyplot as plt
from keras import backend as K
from keras.layers import Lambda, Input
from keras.models import Model
import tensorflow as tf
from subprocess import Popen, PIPE
from timeit import default_timer as timer
from losses import psnr2 as psnr

class Metrics():
    def __init__(self,use_gpu=True):
        '''
        net - ['squeeze','alex','vgg']
        '''

    def psnr_video(self,videopath1, videopath2):
        cap1 = cv2.VideoCapture(videopath1)
        cap2 = cv2.VideoCapture(videopath2)
        psnr_values = []
        while(cap1.isOpened() and cap2.isOpened()):
            ret1, frame1 = cap1.read()
            ret2, frame2 = cap2.read()
            if(ret1 and ret2):
                psnr_values.append(self.calculate_psnr(frame1, frame2))
            else:
                break
        cap1.release()
        cap2.release()
        return np.mean(psnr_values)

    def ssim_video(self,videopath1, videopath2):
        cap1 = cv2.VideoCapture(videopath1)
        cap2 = cv2.VideoCapture(videopath2)
        ssim_values = []
        while(cap1.isOpened() and cap2.isOpened()):
            ret1, frame1 = cap1.read()
            ret2, frame2 = cap2.read()
            if(ret1 and ret2):
                ssim_values.append(self.calculate_ssim(frame1, frame2))
            else:
                break
        cap1.release()
        cap2.release()
        return np.mean(ssim_values)

    # Calculate psnr between two images
    def calculate_psnr(self,img1, img2):
        # img1 and img2 have range [0, 255]
        img1 = img1.astype(np.float64)
        img2 = img2.astype(np.float64)
        mse = np.mean((img1 - img2)**2)
        if mse == 0:
            return float('inf')
        return 20 * math.log10(255.0 / math.sqrt(mse))

    # Aux to Calculate psnr between two images
    def ssim(self,img1, img2):
        C1 = (0.01 * 255)**2
        C2 = (0.03 * 255)**2

        img1 = img1.astype(np.float64)
        img2 = img2.astype(np.float64)
        kernel = cv2.getGaussianKernel(11, 1.5)
        window = np.outer(kernel, kernel.transpose())

        mu1 = cv2.filter2D(img1, -1, window)[5:-5, 5:-5]  # valid
        mu2 = cv2.filter2D(img2, -1, window)[5:-5, 5:-5]
        mu1_sq = mu1**2
        mu2_sq = mu2**2
        mu1_mu2 = mu1 * mu2
        sigma1_sq = cv2.filter2D(img1**2, -1, window)[5:-5, 5:-5] - mu1_sq
        sigma2_sq = cv2.filter2D(img2**2, -1, window)[5:-5, 5:-5] - mu2_sq
        sigma12 = cv2.filter2D(img1 * img2, -1, window)[5:-5, 5:-5] - mu1_mu2

        ssim_map = ((2 * mu1_mu2 + C1) * (2 * sigma12 + C2)) / ((mu1_sq + mu2_sq + C1) *
                                                                (sigma1_sq + sigma2_sq + C2))
        return ssim_map.mean()

    # Calculate ssim between two images
    def calculate_ssim(self,img1, img2):
        '''calculate SSIM
        the same outputs as MATLAB's
        img1, img2: [0, 255]
        '''
        if not img1.shape == img2.shape:
            raise ValueError('Input images must have the same dimensions.')
        if img1.ndim == 2:
            return self.ssim(img1, img2)
        elif img1.ndim == 3:
            if img1.shape[2] == 3:
                ssims = []
                for i in range(3):
                    ssims.append(self.ssim(img1, img2))
                return np.array(ssims).mean()
            elif img1.shape[2] == 1:
                return self.ssim(np.squeeze(img1), np.squeeze(img2))
        else:
            raise ValueError('Wrong input image dimensions.')

class DataLoader(Sequence):
    def __init__(self, datapath, batch_size, height_hr, width_hr, scale, crops_per_image, media_type,time_step):
        """
        :param string datapath: filepath to training images
        :param int height_hr: Height of high-resolution images
        :param int width_hr: Width of high-resolution images
        :param int height_hr: Height of low-resolution images
        :param int width_hr: Width of low-resolution images
        :param int scale: Upscaling factor
        """

        # Store the datapath
        self.datapath = datapath
        self.batch_size = batch_size
        self.height_hr = height_hr
        self.height_lr = int(height_hr / scale)
        self.width_hr = width_hr
        self.width_lr = int(width_hr / scale)
        self.channel = 1
        self.scale = scale
        self.crops_per_image = crops_per_image
        self.media_type  = media_type
        self.time_step=time_step
        self.total_imgs = None

        # Options for resizing
        self.options = [Image.NEAREST, Image.BILINEAR, Image.BICUBIC, Image.LANCZOS]

        # Check data source
        self.img_paths = []

        if os.path.isdir(self.datapath):
            self.get_paths()

    def get_paths(self):
        for dirpath, _, filenames in os.walk(self.datapath):
            for filename in [f for f in filenames if any(filetype in f.lower() for filetype in ['jpeg', 'png', 'jpg','mp4','264','webm','wma'])]:
                self.img_paths.append(os.path.join(dirpath, filename))
        self.total_imgs = len(self.img_paths)
        print(">> Found {} images in dataset".format(self.total_imgs))

    def random_crop(self, img, random_crop_size):
        # Note: image_data_format is 'channel_last'
        assert img.shape[2] == 3
        height, width = img.shape[0], img.shape[1]
        dy, dx = random_crop_size
        x = np.random.randint(0, width - dx + 1)
        y = np.random.randint(0, height - dy + 1)
        return img[y:(y+dy), x:(x+dx), :]

    def fix_crop(self, img, dy, dx, y, x):
        return img[y:(y+dy), x:(x+dx), :]

    @staticmethod
    def scale_lr_imgs(imgs):
        """Scale low-res images prior to passing to SRGAN"""
        return imgs / 255.

    @staticmethod
    def unscale_lr_imgs(imgs):
        """Un-Scale low-res images"""
        return imgs * 255

    @staticmethod
    def scale_hr_imgs(imgs):
        """Scale high-res images prior to passing to SRGAN"""
        return imgs / 127.5 - 1

    @staticmethod
    def unscale_hr_imgs(imgs):
        """Un-Scale high-res images"""
        return (imgs + 1.) * 127.5

    def count_frames_manual(self,cap):
        count=0
        while(cap.isOpened()):
            ret, frame = cap.read()
            if(ret):
                count +=1
            else:
                break
        return count

    def count_frames(self,cap):
        '''Count total frames in video'''
        try:
            total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
        except:
            total = self.count_frames_manual(cap)
        return total

    def get_random_frames(self,n_fms=1,cap=None,time_step=1):
        """Get random number of video frames"""
        np.random.seed(int(1000000*(timer()%1)))
        t_frames = self.count_frames(cap)
        if(time_step==1):
            choiced_frames = np.random.randint(t_frames, size=n_fms)
        else:
            choiced_frames = np.random.randint(t_frames-time_step, size=n_fms)
        return choiced_frames

    @staticmethod
    def load_img(path):
        img = Image.open(path)
        if img.mode != 'RGB':
            img = img.convert('RGB')
        return np.array(img)

    def load_frame(self,videopath,time_step=1,colorspace='YCbCr'):
        """Get n_imgs random frames from the video"""
        cap = cv2.VideoCapture(videopath)
        if(not cap.isOpened()):
            print("Error to open video: ",videopath)
            return -1
        choiced_frame=self.get_random_frames(1,cap,time_step)
        cap.set(1,choiced_frame)
        frames = []
        for i in list(range(time_step)):
            ret, frame = cap.read()
            if not ret:
                print(">> Erro to access frames.")
                return -1
            if colorspace=='YCbCr':
                frame = cv2.cvtColor(frame, cv2.COLOR_BGR2YCrCb)[:,:,:self.channel]
                frames.append(frame)
            else:
                frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
        cap.release()
        return np.array(frames)

    def resize(self,shape,scale,image):

        # 3 channel images of arbitrary shape
        inp = Input(shape=shape)
        try:
            out = Lambda(lambda image: K.tf.image.resize_bicubic(image, (shape[0]//scale, shape[1]//scale)))(inp)
        except :
            # if you have older version of tensorflow
            out = Lambda(lambda image: K.tf.image.resize_bicubic(image, shape[0]//scale, shape[1]//scale))(inp)
        model = Model(inputs=inp, outputs=out)
        out = model.predict(image[np.newaxis, ...])
        return np.squeeze(out, axis=0).astype(np.uint8)

    def __len__(self):
        return int(self.total_imgs / float(self.batch_size))

    def __getitem__(self, idx):
        return self.load_batch(idx=idx)

    def load_batch(self,idx=0, img_paths=None, training=True, bicubic=False):
        """ Loads a batch of images or video"""
        if(self.media_type=='i'):
            #print("1. Media type image folder")
            imgs_lr, imgs_hr = self.load_batch_image(idx, img_paths=img_paths, training=training,bicubic=False)
        elif(self.media_type=='v' and os.path.isdir(self.datapath)):
            #print("2. Media type video folder")
            imgs_lr, imgs_hr = self.load_batch_video(idx, img_paths=None, training=training, bicubic=False)
        elif(self.media_type=='v' and not os.path.isdir(self.datapath)):
            #print("3. Media type single video")
            imgs_lr, imgs_hr = self.load_batch_video(idx, img_paths=self.datapath, training=training, bicubic=False)
        elif(self.media_type=='vt' and os.path.isdir(self.datapath)):
            #print("4. Media type single video")
            imgs_lr, imgs_hr = self.load_temporal_batch_video(idx, img_paths=None, training=training, bicubic=False)
        else:
            print("Error: type of media fail")
        return imgs_lr, imgs_hr

    def load_temporal_batch_video(self, idx=0, img_paths=None, training=True, bicubic=False):
        """Loads a batch of frames from videos folder"""
        # Starting index to look in
        cur_idx=np.random.randint(self.total_imgs)
        if not img_paths:
            cur_idx = idx*self.batch_size
        #print('cur_idx:',cur_idx)
        if training == False:
            cur_idx = idx

        # Scale and pre-process images
        imgs_hr, imgs_lr = [], []
        for i in range(self.time_step):
            imgs_lr.append([])
        while True:
            # Check if done with batch
            if img_paths is None:
                if cur_idx >= self.total_imgs:
                    cur_idx = 0
                if len(imgs_hr) >= self.batch_size:
                    break
            if img_paths is not None and len(imgs_hr) == len(img_paths):
                break
            try:
                # Load image
                ori_imgs_hr = None
                if img_paths:
                    ori_imgs_hr = self.load_frame(img_paths[cur_idx],self.time_step)
                else:
                    ori_imgs_hr = self.load_frame(self.img_paths[cur_idx],self.time_step)

                # Create HR images to go through
                img_crops = []
                if training:
                    for i in range(self.crops_per_image):
                        assert ori_imgs_hr.shape[3] == self.channel
                        height, width = ori_imgs_hr.shape[1], ori_imgs_hr.shape[2]
                        x = np.random.randint(0, width - self.width_hr + 1)
                        y = np.random.randint(0, height - self.height_hr + 1)
                        t_step_img_crops = []
                        for img_hr in ori_imgs_hr:
                            t_step_img_crops.append(self.fix_crop(img_hr,self.height_hr,self.width_hr,y,x))
                        img_crops.append(t_step_img_crops)
                else:
                    assert ori_imgs_hr.shape[3] == self.channel
                    t_step_img_crops = []
                    for img_hr in ori_imgs_hr:
                       t_step_img_crops.append(img_hr)
                    img_crops.append(t_step_img_crops)

                # Downscale the HR images and save
                for t_step_img_hr in img_crops:
                    t_step_imgs_lr = []
                    if img_paths is None:
                        if cur_idx >= self.total_imgs:
                            cur_idx = 0
                        if len(imgs_hr) >= self.batch_size:
                            break
                    if img_paths is not None and len(imgs_hr) == len(img_paths):
                        break
                    i=0
                    for img_hr in t_step_img_hr:
                        # For LR, do bicubic downsampling
                        method = Image.BICUBIC if bicubic else choice(self.options)
                        lr_shape = (int(img_hr.shape[1]/self.scale), int(img_hr.shape[0]/self.scale))
                        img_lr = Image.fromarray(img_hr.astype(np.uint8))
                        img_lr = np.array(img_lr.resize(lr_shape, method))
                        # Scale color values
                        img_hr = self.scale_hr_imgs(img_hr)
                        img_lr = self.scale_lr_imgs(img_lr)

                        # Store LR images
                        imgs_lr[i].append(img_lr)
                        i+=1
                    # Store LR and HR images
                    imgs_hr.append(np.expand_dims(img_hr, axis=0))
            except Exception as e:
                print(e)
                pass
            finally:
                cur_idx += 1

        # Convert to numpy arrays when we are training
        # Note: all are cropped to same size, which is not the case when not training
        for i in range(self.time_step):
            imgs_lr[i] = np.array(imgs_lr[i])
        imgs_hr = np.array(imgs_hr)
        imgs_hr=imgs_hr.reshape(self.batch_size,img_hr.shape[0],img_hr.shape[1],self.channel)

        # Return image batch
        return imgs_lr, imgs_hr

    def load_batch_video(self, idx=0, img_paths=None, training=True, bicubic=False):
        """Loads a batch of frames from videos folder"""
        # Starting index to look in
        cur_idx = 0
        if not img_paths:
            cur_idx = idx*self.batch_size
        #print('cur_idx:',cur_idx)

        # Scale and pre-process images
        imgs_hr, imgs_lr = [], []
        while True:

            # Check if done with batch
            if img_paths is None:
                if cur_idx >= self.total_imgs:
                    cur_idx = 0
                if len(imgs_hr) >= self.batch_size:
                    break
            if img_paths is not None and len(imgs_hr) == len(img_paths):
                break

            try:
                # Load image
                img_hr = None
                if img_paths:
                    #print('1. video: ',img_paths[cur_idx])
                    img_hr = self.load_frame(img_paths[cur_idx])
                    #print(img_hr.shape)
                else:
                    #print('2. video: ',self.img_paths[cur_idx])
                    img_hr = self.load_frame(self.img_paths[cur_idx])
                    #print(img_hr.shape)

                # Create HR images to go through
                img_crops = []
                if training:
                    for i in range(self.crops_per_image):
                        #print(idx, cur_idx, "Loading crop: ", i)
                        img_crops.append(self.random_crop(img_hr, (self.height_hr, self.width_hr)))
                else:
                    img_crops = [img_hr]

                # Downscale the HR images and save
                for img_hr in img_crops:

                    # TODO: Refactor this so it does not occur multiple times
                    if img_paths is None:
                        if cur_idx >= self.total_imgs:
                            cur_idx = 0
                        if len(imgs_hr) >= self.batch_size:
                            break
                    if img_paths is not None and len(imgs_hr) == len(img_paths):
                        break

                    # For LR, do bicubic downsampling
                    method = Image.BICUBIC if bicubic else choice(self.options)
                    lr_shape = (int(img_hr.shape[1]/self.scale), int(img_hr.shape[0]/self.scale))
                    img_lr = Image.fromarray(img_hr.astype(np.uint8))
                    img_lr = np.array(img_lr.resize(lr_shape, method))

                    # Scale color values
                    img_hr = self.scale_hr_imgs(img_hr)
                    img_lr = self.scale_lr_imgs(img_lr)

                    # Store images
                    imgs_hr.append(img_hr)
                    imgs_lr.append(img_lr)

            except Exception as e:
                print(e)
                pass
            finally:
                cur_idx += 1

        # Convert to numpy arrays when we are training
        # Note: all are cropped to same size, which is not the case when not training
        if training:
            imgs_hr = np.array(imgs_hr)
            imgs_lr = np.array(imgs_lr)

        # Return image batch
        return imgs_lr, imgs_hr

    def load_batch_image(self, idx=0, img_paths=None, training=True, bicubic=False):
        """Loads a batch of images from datapath folder"""

        # Starting index to look in
        cur_idx = 0

        if not img_paths:
            cur_idx = idx*self.batch_size

        # Scale and pre-process images
        imgs_hr, imgs_lr = [], []
        while True:

            # Check if done with batch
            if img_paths is None:
                if cur_idx >= self.total_imgs:
                    cur_idx = 0
                if len(imgs_hr) >= self.batch_size:
                    break
            if img_paths is not None and len(imgs_hr) == len(img_paths):
                break

            try:
                # Load image
                img_hr = None
                if img_paths:
                    img_hr = self.load_img(img_paths[cur_idx])
                else:
                    img_hr = self.load_img(self.img_paths[cur_idx])

                # Create HR images to go through
                img_crops = []
                if training:
                    for i in range(self.crops_per_image):
                        #print(idx, cur_idx, "Loading crop: ", i)
                        img_crops.append(self.random_crop(img_hr, (self.height_hr, self.width_hr)))
                else:
                    img_crops = [img_hr]

                # Downscale the HR images and save
                for img_hr in img_crops:

                    # TODO: Refactor this so it does not occur multiple times
                    if img_paths is None:
                        if cur_idx >= self.total_imgs:
                            cur_idx = 0
                        if len(imgs_hr) >= self.batch_size:
                            break
                    if img_paths is not None and len(imgs_hr) == len(img_paths):
                        break

                    # For LR, do bicubic downsampling
                    method = Image.BICUBIC if bicubic else choice(self.options)
                    lr_shape = (int(img_hr.shape[1]/self.scale), int(img_hr.shape[0]/self.scale))
                    img_lr = Image.fromarray(img_hr.astype(np.uint8))
                    img_lr = np.array(img_lr.resize(lr_shape, method))

                    # Scale color values
                    img_hr = self.scale_hr_imgs(img_hr)
                    img_lr = self.scale_lr_imgs(img_lr)

                    # Store images
                    imgs_hr.append(img_hr)
                    imgs_lr.append(img_lr)

            except Exception as e:
                print(e)
                pass
            finally:
                cur_idx += 1

        # Convert to numpy arrays when we are training
        # Note: all are cropped to same size, which is not the case when not training
        if training:
            imgs_hr = np.array(imgs_hr)
            imgs_lr = np.array(imgs_lr)

        # Return image batch
        return imgs_lr, imgs_hr

    def sr_genarator(self,model,img_lr):
        """Predict sr frame given a LR frame"""
        # Predict high-resolution version (add batch dimension to image)
        img_lr=self.scale_lr_imgs(img_lr)
        img_sr = model.generator.predict(np.expand_dims(img_lr, 0))
        # Remove batch dimension
        img_sr = img_sr.reshape(img_sr.shape[1], img_sr.shape[2], img_sr.shape[3])
        img_sr = self.unscale_hr_imgs(img_sr)
        return img_sr

    def write_sr_images(self,model, input_images, output_images):
        """
        :param SRGAN model: The trained SRGAN model
        :param list input_images: List of filepaths for input images
        :param list output_images: List of filepaths for output images
        """

        # Load the images to perform test on images
        imgs_lr, imgs_hr = self.load_batch(idx=0,img_paths=input_images, training=False)
        # Scale color values
        imgs_hr = self.unscale_hr_imgs(np.array(imgs_hr))
        imgs_lr = self.unscale_lr_imgs(np.array(imgs_lr))

        # Create super resolution images
        imgs_sr = []
        time_elapsed = []
        for img_lr,img_hr in zip(imgs_lr,imgs_hr):
            start = timer()
            img_sr = self.sr_genarator(model,img_lr)
            end = timer()
            time_elapsed.append(end - start)

            img_sr = Image.fromarray(img_sr.astype(np.uint8))
            img_sr.save(output_images.split(".")[0]+"SR.png")
            #imageio.imwrite(output_images.split(".")[0]+"SR.png", img_sr)

            img_hr = Image.fromarray(img_hr.astype(np.uint8))
            img_hr.save(output_images.split(".")[0]+"HR.png")
            #imageio.imwrite(output_images.split(".")[0]+"HR.png", img_hr)

            img_lr = Image.fromarray(img_lr.astype(np.uint8))
            img_lr.save(output_images.split(".")[0]+"LR.png")
            #imageio.imwrite(output_images.split(".")[0]+"LR.png", img_lr)
        return time_elapsed

class VideoRestore():
    #def __init__(self):

    @staticmethod
    def scale_lr_imgs(imgs):
        """Scale low-res images prior to passing to SRGAN"""
        return imgs / 255.

    @staticmethod
    def unscale_hr_imgs(imgs):
        """Un-Scale high-res images"""
        return (imgs + 1.) * 127.5

    def count_frames_manual(self,cap):
        count=0
        while(cap.isOpened()):
            ret, frame = cap.read()
            if(ret):
                count +=1
            else:
                break
        return count

    def count_frames(self,cap):
        '''Count total frames in video'''
        try:
            total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
        except:
            total = self.count_frames_manual(cap)
        return total

    def sr_genarator(self,model,img_lr):
        """Predict sr frame given a LR frame"""
        # Predict high-resolution version (add batch dimension to image)
        img_sr = np.squeeze(
                    model.generator.predict(img_lr,
                        batch_size=1
                    ),
                    axis=0
                )
        # Remove batch dimension
        img_sr = self.unscale_hr_imgs(img_sr)
        return img_sr

    def write_srvideo(self, model=None,lr_videopath=None,sr_videopath=None,print_frequency=30,crf=15):
        """Predict SR video given LR video """
        cap = cv2.VideoCapture(lr_videopath)
        if cap.isOpened():
            fps = math.ceil(cap.get(cv2.CAP_PROP_FPS))
            # ffmpeg setup '-qscale', '5',
            p = Popen(['ffmpeg', '-y', '-f', 'image2pipe', '-vcodec', 'mjpeg', '-r', str(fps), '-i', '-', '-vcodec', 'libx264','-preset', 'veryslow', '-crf',str(crf), '-r', str(fps), sr_videopath], stdin=PIPE)
        else:
            print("Error to open low resolution video")
            return -1

        # Get video total frames
        t_frames = self.count_frames(cap)
        #cria arquivo video hr if hr video is open
        count = 0
        time_elapsed = []
        while cap.isOpened():
            ret, frame = cap.read()
            if ret:
                frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
                start = timer()
                img_sr = self.sr_genarator(model,frame)
                end = timer()
                time_elapsed.append(end - start)
                im = Image.fromarray(img_sr.astype(np.uint8))
                im.save(p.stdin, 'JPEG')
                count +=1
            else:
                break
            if(count % print_frequency == 0):
                print('Time per Frame: '+str(np.mean(time_elapsed))+'s')
                print('Estimated time: '+str(np.mean(time_elapsed)*(t_frames-count)/60.)+'min')
        p.stdin.close()
        p.wait()
        cap.release()
        return time_elapsed

    def write_temporal_srvideo(self, model=None,lr_videopath=None,sr_videopath=None,print_frequency=30,crf=15,time_step=1):
        """Predict SR video given LR video """
        cap = cv2.VideoCapture(lr_videopath)
        if cap.isOpened():
            fps = math.ceil(cap.get(cv2.CAP_PROP_FPS))
            # ffmpeg setup '-qscale', '5',
            p = Popen(['ffmpeg', '-y', '-f', 'image2pipe', '-vcodec', 'mjpeg', '-r', str(fps), '-i', '-', '-vcodec', 'libx264','-preset', 'veryslow', '-crf',str(crf), '-r', str(fps), sr_videopath], stdin=PIPE)
        else:
            print("Error to open low resolution video")
            return -1

        # Get video total frames
        t_frames = self.count_frames(cap)
        #cria arquivo video hr if hr video is open
        count = 0
        time_elapsed = []
        frame_steps = []
        cap.set(1,0)
        while cap.isOpened() and count < t_frames:
            ret, frame = cap.read()
            if ret:
                frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
                if count > 0:
                    frame_steps.append(np.expand_dims(self.scale_lr_imgs(frame),0))
                    frame_steps.pop(0)
                else:
                    frame_steps = [np.expand_dims(self.scale_lr_imgs(frame),0) for f in range(time_step)]
                start = timer()
                img_sr = self.sr_genarator(model,frame_steps)
                end = timer()
                time_elapsed.append(end - start)
                im = Image.fromarray(img_sr.astype(np.uint8))
                im.save(p.stdin, 'JPEG')
                count +=1
            else:
                break
            if(count % print_frequency == 0):
                print('Time per Frame: '+str(np.mean(time_elapsed))+'s')
                print('Estimated time: '+str(np.mean(time_elapsed)*(t_frames-count)/60.)+'min')
        p.stdin.close()
        p.wait()
        cap.release()
        return time_elapsed

def plot_test_images(model, loader, datapath_test, test_output, epoch, name='SRGAN'):
    """
    :param SRGAN model: The trained SRGAN model
    :param DataLoader loader: Instance of DataLoader for loading images
    :param str datapath_test: path to folder with testing images
    :param string test_output: Directory path for outputting testing images
    :param int epoch: Identifier for how long the model has been trained
    """

    try:
        # Get the location of test images
        test_images = [os.path.join(datapath_test, f) for f in os.listdir(datapath_test) if any(filetype in f.lower() for filetype in ['jpeg','mp4','264', 'png', 'jpg'])]

        # Load the images to perform test on images
        imgs_lr, imgs_hr = loader.load_batch(img_paths=test_images, training=False, bicubic=True)

        # Create super resolution and bicubic interpolation images
        imgs_sr = []
        imgs_bc = []
        for i in range(len(test_images)):

            # Bicubic interpolation
            pil_img = loader.unscale_lr_imgs(imgs_lr[i]).astype('uint8')
            pil_img = Image.fromarray(pil_img)
            hr_shape = (imgs_hr[i].shape[1], imgs_hr[i].shape[0])

            imgs_bc.append(
                loader.scale_lr_imgs(
                    np.array(pil_img.resize(hr_shape, resample=Image.BICUBIC))
                )
            )

            # SRGAN prediction
            imgs_sr.append(
                np.squeeze(
                    model.predict(
                        np.expand_dims(imgs_lr[i], 0),
                        batch_size=1
                    ),
                    axis=0
                )
            )

        # Unscale colors values
        imgs_lr = [loader.unscale_lr_imgs(img).astype(np.uint8) for img in imgs_lr]
        imgs_hr = [loader.unscale_hr_imgs(img).astype(np.uint8) for img in imgs_hr]
        imgs_sr = [loader.unscale_hr_imgs(img).astype(np.uint8) for img in imgs_sr]

        # Loop through images
        for img_hr, img_lr, img_bc, img_sr, img_path in zip(imgs_hr, imgs_lr, imgs_bc, imgs_sr, test_images):

            # Get the filename
            filename = os.path.basename(img_path).split(".")[0]

            # Images and titles
            images = {
                'Low Resolution': img_lr,
                'Bicubic Interpolation': img_bc,
                name: img_sr,
                'Original': img_hr
            }

        # Plot the images. Note: rescaling and using squeeze since we are getting batches of size 1
            fig, axes = plt.subplots(1, 4, figsize=(40, 10))
            for i, (title, img) in enumerate(images.items()):
                axes[i].imshow(img)
                axes[i].set_title("{} - {}".format(title, img.shape))
                axes[i].axis('off')
            plt.suptitle('{} - Epoch: {}'.format(filename, epoch))

            # Save directory
            savefile = os.path.join(test_output, "{}-Epoch{}.png".format(filename, epoch))
            fig.savefig(savefile)
            plt.close()
            gc.collect()
    except Exception as e:
        print(e)

def plot_temporal_test_images(model, loader, datapath_test, test_output, epoch, name='TSRGAN',time_step=3):
    """
    :param SRGAN model: The trained SRGAN model
    :param DataLoader loader: Instance of DataLoader for loading images
    :param str datapath_test: path to folder with testing images
    :param string test_output: Directory path for outputting testing images
    :param int epoch: Identifier for how long the model has been trained
    """

    try:
        # Get the location of test images
        test_images = [os.path.join(datapath_test, f) for f in os.listdir(datapath_test) if any(filetype in f.lower() for filetype in ['mp4','webm','264'])]

        # Create super resolution and bicubic interpolation images
        imgs_sr = []
        imgs_bc = []
        _imgs_lr = []
        _imgs_hr = []
        for i in range(len(test_images)):
            # Load the images to perform test on images
            imgs_lr, imgs_hr = loader.load_batch(idx=i,training=True, bicubic=True)

            # Bicubic interpolation
            pil_img = loader.unscale_lr_imgs(np.squeeze(imgs_lr[time_step-1], axis=0)).astype('uint8')
            pil_img = Image.fromarray(pil_img)
            hr_shape = (imgs_hr.shape[2], imgs_hr.shape[1])

            _imgs_lr.append(np.squeeze(imgs_lr[time_step-1], axis=0))
            _imgs_hr.append(np.squeeze(imgs_hr, axis=0))
            imgs_bc.append(
                loader.scale_lr_imgs(
                    np.array(pil_img.resize(hr_shape, resample=Image.BICUBIC))
                )
            )

            # TSRGAN prediction
            imgs_sr.append(
                np.squeeze(
                    model.generator.predict(imgs_lr,
                        batch_size=1
                    ),
                    axis=0
                )
            )

        # Unscale colors values
        imgs_lr = [loader.unscale_lr_imgs(img).astype(np.uint8) for img in _imgs_lr]
        imgs_hr = [loader.unscale_hr_imgs(img).astype(np.uint8) for img in _imgs_hr]
        imgs_sr = [loader.unscale_hr_imgs(img).astype(np.uint8) for img in imgs_sr]

        # Loop through images
        for img_hr, img_lr, img_bc, img_sr, img_path in zip(imgs_hr, imgs_lr, imgs_bc, imgs_sr, test_images):

            # Get the filename
            filename = os.path.basename(img_path).split(".")[0]

            # Images and titles
            images = {
                'Low Resolution': [img_lr, img_hr],
                'Bicubic Interpolation': [img_bc,img_hr],
                name: [img_sr,img_hr],
                'Original': [img_hr,None]
            }

            # Plot the images. Note: rescaling and using squeeze since we are getting batches of size 1
            fig, axes = plt.subplots(1, 4, figsize=(40, 10))
            for i, (title, imgs) in enumerate(images.items()):
                axes[i].imshow(imgs[0])
                axes[i].set_title("{} - {} {}".format(title, imgs[0].shape, ("- psnr: "+str(round(psnr(imgs[1],imgs[0]),2)) if (title == name or title == 'Bicubic Interpolation') else " ")))
                axes[i].axis('off')
            plt.suptitle('{} - Epoch: {}'.format(filename, epoch))

            # Save directory
            savefile = os.path.join(test_output, "{}-Epoch{}.png".format(filename, epoch))
            fig.savefig(savefile)
            plt.close()
            gc.collect()
    except Exception as e:
        print(e)

def PSNR(y_true, y_pred):
        """
        PSNR is Peek Signal to Noise Ratio, see https://en.wikipedia.org/wiki/Peak_signal-to-noise_ratio

        The equation is:
        PSNR = 20 * log10(MAX_I) - 10 * log10(MSE)

        Since input is scaled from -1 to 1, MAX_I = 1, and thus 20 * log10(1) = 0. Only the last part of the equation is therefore neccesary.
        """
        return -10.0 * K.log(K.mean(K.square(y_pred - y_true))) / K.log(10.0)


"""
There are two versions of detection layer in darknet, one of them is named `region`.
This module contains this layer implemented.
"""

from keras import backend as K
from keras.engine.topology import Layer
import numpy as np
#from builtins import range
from keras.activations import softmax

class Region(Layer):
    def __init__(self, coords=4, classes=20, num=1,
            log=0, sqrt=0, softmax=0, background=0, max=30,
            jitter=0.2,
            rescore = 0, thresh=0.5, classfix=0, absolute=0, random=0,
            coord_scale=1, object_scale=1,
            noobject_scale=1, class_scale=1,
            bias_match=0,
            tree=None,#tree_file for softmax_tree - not used now
            map_filename=None, # file name for map_file - not used
            anchors=None,
            **kwargs
            ):
        super(Region, self).__init__(**kwargs)
        self.coords = coords
        self.classes = classes
        self.num = num
        self.background = background
        print(coords, classes)
        self.c = (self.coords+self.classes+1)*num
        if anchors:
            self.biases = list(map(float, anchors))
        pass

    def build(self, input_shape):
        self.inputs = np.prod(input_shape[1:])
        self.outputs = self.inputs
        #assert(self.c == input_shape[-1])
        pass

    def _entry_index(self, n, entry):
        #simplified version - gets last coordinate in input tensor
        return n*(self.coords + self.classes + 1) + entry

    def _process_input(self, x):
        """Apply logistic and softmax activations to input tensor
        """
        logistic_activate = lambda x: 1.0/(1.0 + K.exp(-x))

        (batch, w, h, channels) = x.get_shape()
        x_temp = K.permute_dimensions(x, (3, 0, 1, 2))
        x_t = []
        for i in range(self.num):
            k = self._entry_index(i, 0)
            x_t.extend([
                logistic_activate(K.gather(x_temp, (k, k + 1))), # 0
                K.gather(x_temp, (k + 2, k + 3))])
            if self.background:
                x_t.append(K.gather(x_temp, (k + 4,)))
            else:
                x_t.append(logistic_activate(K.gather(x_temp, (k + 4,))))

            x_t.append(
                softmax(
                    K.gather(x_temp, tuple(range(k + 5, k + self.coords + self.classes + 1))),
                    axis=0))
        x_t = K.concatenate(x_t, axis=0)
        return K.permute_dimensions(x_t, (1, 2, 3, 0))

    def call(self, x, training=None):
        #(w, h, channels) = x.get_shape()[1:]
        return self._process_input(x)
        #return x

    def compute_output_shape(self, input_shape):
        return input_shape


import pandas as pd                                                     # manipulação de dados
import matplotlib.pyplot as plt                                         # visualização de dados
import seaborn as sns                                                   # visualização de dados
from sklearn.feature_extraction.text import CountVectorizer             # machine learn (Cálculos)

def Resposta02(connection):
    print("\nPergunta 02: \n")
    consultaPerg02 = "Select genres, Count(*) From titles Where type = 'movie' Group By genres"
    resultadoPerg02 = pd.read_sql_query(consultaPerg02, connection)
    resultadoPerg02['genres'] = resultadoPerg02['genres'].str.lower().values  # Convertendo valores para minusculo
    valoresSemNA = resultadoPerg02['genres'].dropna()  # Removendo valores N/A

    padrao = "(?u)\\b[\\w-]+\\b"    # Expressão Regular
    vetor = CountVectorizer(token_pattern=padrao, analyzer='word').fit(
        valoresSemNA)  # Criando vetor para contagem precisa

    bagGeneros = vetor.transform(valoresSemNA)  # Armazenando os valores
    generosValores = vetor.get_feature_names()  # Pegando os valores do genero
    generos = pd.DataFrame(bagGeneros.todense(), columns=generosValores,
                           index=valoresSemNA.index)  # Criando um DataFrame de visualização
    # print(generos.info())
    generos = generos.drop(columns='n', axis=0)  # Drop na coluna N, criada errado
    generosPercentual = pd.Series(generos.sum()).sort_values(ascending=False) / generos.shape[0] * 100
    print(generosPercentual.head(10))  # Top 10 generos com %

    # Gerar gráfico
    plt.figure(figsize=(16, 8))
    sns.barplot(x=generosPercentual.values, y=generosPercentual.index, orient="h",
                palette="terrain")  # Criação do gráfico de barras
    plt.ylabel("Gênero")
    plt.xlabel("\nPercentual de Filmes (%)")
    plt.title("\nNúmero de Títulos por Gênero\n")
    plt.show()


'''
machine learning models for time series forecasting
'''

from sklearn.svm import SVR
from sklearn.ensemble import RandomForestRegressor
from sklearn.tree import DecisionTreeRegressor
from hmmlearn.hmm import GaussianHMM, GMMHMM
from sklearn.linear_model import LinearRegression
from src import eval
from src.util import *
from sklearn.preprocessing import MinMaxScaler
from src.ts_decompose import ts_decompose

# train ML models
def train_ML_model(model, trainX, trainY):

    n = trainX.shape[0]
    print("trainx num is:", n)
    model.fit(trainX, trainY)

    return model

# predict testing set using trained model
def predict_ML_model(model, testX):

    n = testX.shape[0]
    print("testx num is:", n)
    testy = model.predict(testX)

    return testy

# predict testing set for multi-step ahead forecasting, using iteration method
def predict_ML_model_iteration(model, testX, look_ahead):

    testBatchSize = testX.shape[0]

    ans = []

    for i in range(look_ahead):

        pred = model.predict(testX).flatten()  # (test_num, )
        ans.append(pred)

        testX = testX[:, 1:]  # drop the head
        pred = pred.reshape((testBatchSize, 1))  # '1' represents the one-step ahead forecasting
        testX = np.append(testX, pred, axis=1)  # add the prediction results to the tail

    ans = np.array(ans)
    ans = ans.transpose([1, 0])
    return ans

def ML_forecasting(data, model, lag, train_look_ahead, test_look_ahead):

    scaler = MinMaxScaler(feature_range=(0, 1))
    dataset = scaler.fit_transform(data)

    # 分割序列为样本
    trainData, testData = divideTrainTest(dataset)

    flag = False
    trainX, trainY = create_multi_ahead_samples(trainData, lag, lookAhead=train_look_ahead, RNN=flag)
    testX, testY = create_multi_ahead_samples(testData, lag, lookAhead=test_look_ahead, RNN=flag)
    print("testX shape:", testX.shape)
    print("testy shape:", testY.shape)
    print("trainX shape:", trainX.shape)
    print("trainy shape:", trainY.shape)

    model = train_ML_model(model, trainX, trainY)

    testPred = predict_ML_model_iteration(model, testX, test_look_ahead)
    print("testPred shape:", testPred.shape)

    testPred = scaler.inverse_transform(testPred)
    testY = scaler.inverse_transform(testY)

    MAE = eval.calcMAE(testY, testPred)
    print("test MAE", MAE)
    MRSE = eval.calcRMSE(testY, testPred)
    print("test RMSE", MRSE)
    SMAPE = eval.calcSMAPE(testY, testPred)
    print("test SMAPE", SMAPE)

    return testPred, testY

def decompose_ML_forecasting(model, lag, freq, h_train, h_test):

    # 序列分解
    trend, seasonal, residual = ts_decompose(ts, freq)
    print("ts decomposition is finished!")
    print("trend shape is", trend.shape)
    print("season shape is", seasonal.shape)
    print("residual shape is", residual.shape)

    # forecasting sub-series independently
    trend_pred, trend_y = ML_forecasting(trend, model, lag=lag,  train_look_ahead=h_train, test_look_ahead=h_test)
    res_pred, res_y = ML_forecasting(residual, model, lag=lag,  train_look_ahead=h_train, test_look_ahead=h_test)
    season_pred, season_y = ML_forecasting(seasonal, model, lag=lag,  train_look_ahead=h_train, test_look_ahead=h_test)

    trend_pred = trend_pred.reshape(-1, h_test)
    trend_y = trend_y.reshape(-1, h_test)
    res_pred = res_pred.reshape(-1, h_test)
    res_y = res_y.reshape(-1, h_test)
    season_pred = season_pred.reshape(-1, h_test)
    season_y = season_y.reshape(-1, h_test)

    print("trend_pred shape is", trend_pred.shape)
    print("res_pred shape is", res_pred.shape)
    print("season_pred shape is", season_pred.shape)
    print("trend_y shape is", trend_y.shape)
    print("res_y shape is", res_y.shape)
    print("season_y shape is", season_y.shape)

    testPred = trend_pred + res_pred + season_pred
    testY = trend_y + res_y + season_y

    # evaluation
    MAE = eval.calcMAE(testY, testPred)
    print("test MAE", MAE)
    MRSE = eval.calcRMSE(testY, testPred)
    print("test RMSE", MRSE)
    MAPE = eval.calcMAPE(testY, testPred)
    print("test MAPE", MAPE)
    SMAPE = eval.calcSMAPE(testY, testPred)
    print("test SMAPE", SMAPE)

    return testPred, testY

if __name__ == '__main__':

    # parameters
    lag = 24
    h_train = 1
    h_test = 1
    freq = 8
    print("lag:", lag)
    print("train look ahead:", h_train)
    print("test look ahead:", h_test)
    print("freq:", freq)

    # model
    # model = SVR(C=0.05, epsilon=0.01, kernel="rbf")
    model = RandomForestRegressor(n_estimators=30, max_depth=6, n_jobs=7)
    # model = DecisionTreeRegressor()
    # model = GaussianHMM(n_components=4)
    # model = LinearRegression()

    # datasets
    ts, data = load_data("../data/NSW2013.csv", columnName="TOTALDEMAND")
    # ts, data = load_data("../data/bike_hour.csv", columnName="cnt")
    # ts, data = load_data("../data/TAS2016.csv", columnName="TOTALDEMAND")
    # ts, data = load_data("../data/traffic_data_in_bits.csv", columnName="value")
    # ts, data = load_data("../data/beijing_pm25.csv", columnName="pm2.5")
    # ts, data = load_data("../data/pollution.csv", columnName="Ozone")

    # training and testing
    # testPred, testY = ML_forecasting(data, model, lag, train_look_ahead=h_train, test_look_ahead=h_test)
    testPred, testY = decompose_ML_forecasting(model, lag, freq, h_train, h_test)


import pandas as pd
import cv2 as cv
import numpy as np

# 将训练集的x和y拆分出来
def separate_xy():
    path = 'initial_data\\train.csv'
    df = pd.read_csv(path)
    df_y = df[['label']]
    df_x = df[['feature']]
    df_y.to_csv('initial_data\\label.csv', index=False, header=False)
    df_x.to_csv('initial_data\\data.csv', index=False, header=False)

separate_xy()
print('按任意键继续------')
input()
print('-----------------')

# 将x可视化
def visualize():
    path = './/image//'
    data = np.loadtxt('initial_data\\data.csv')
    for i in range(data.shape[0]):
        face_array = np.reshape(data[i, :], (48, 48))
        cv.imwrite(path + '//' + '{}.jpg'.format(i), face_array)

visualize()


import pyttsx3
import speech_recognition as sr
import datetime
import wikipedia
import webbrowser
import os
import random
import smtplib
from email.message import EmailMessage

contact = {'anchal': 'anchalgupta1203@gamil.com', 'aditi': 'aditigupta0852@gamil.com', 'isha': 'agen20cs304011@gmail.com'}

engine = pyttsx3.init('sapi5')
voices = engine.getProperty('voices')
engine.setProperty('voice', voices[1].id)

def speak(audio):
    engine.say(audio)
    engine.runAndWait()

def greetMe():
    """
    It greets the user as per the time
    """
    hour = int(datetime.datetime.now().hour)
    if hour >= 0 and hour < 12:
        speak("Good Morning!")

    elif hour >= 12 and hour < 18:
        speak("Good Afternoon!")

    elif hour >= 18 or hour < 0:
        speak("Good Evening!")

    speak(f"I am rose, your assistant please tell me how may i help you?")

def takeCommand():
    """
    It takes microphone input from the user and returns a string as an output
    """

    r = sr.Recognizer()
    with sr.Microphone() as source:
        print("Listening...")
        r.pause_threshold = 1
        audio = r.listen(source)

    try:
        print("Recognizing...")
        query = r.recognize_google(audio,language = 'en-in')
        print(f"user said: {query}\n")

    except Exception as e:
        print("Pardon me, can you please say that again....")
        return "None"
    return query

def sendEmail(to, subject, content):
    server = smtplib.SMTP('smtp.gmail.com', 587)
    server.ehlo()
    server.starttls()
    server.login('anchalgupta1203@gmail.com', 'xdirpguoqneomtuh')
    email = EmailMessage()
    email['From'] = 'anchalgupta1203@gmail.com'
    email['To'] = to
    email['subject'] = subject
    email.set_content(content)
    server.send_message(email)
    server.close()

if __name__ == "__main__":
    greetMe()

# Logic for exceuting tasks based on query
while True:
        query = takeCommand().lower()
        if 'wikipedia' in query:
            speak('Searching wikipedia....')
            query = query.replace("wikipedia", "")
            results = wikipedia.summary(query, sentences=2)
            speak("According to wikipedia")
            print(results)
            speak(results)

        elif 'open youtube' in query:
            webbrowser.open("youtube.com")

        elif 'open google' in query:
            webbrowser.open("google.com")

        elif 'open stake overflow' in query:
            webbrowser.open("stakeoverflow.com")

        elif 'play music' in query:
            music_dir = 'C:\\My_music'
            songs = os.listdir(music_dir)
            index = random.randint(0, len(songs)-1)
            os.startfile(os.path.join(music_dir, songs[index]))

        elif 'time' in query:
            strfTime = datetime.datetime.now().strftime("%H:%M:%S")
            speak(f"Currently the time is {strfTime}")

        elif 'open code' in query:
            codePath = "C:\\Users\\RanuIshu\\AppData\\Local\\Programs\\Microsoft VS Code\\Code.exe"
            os.startfile(codePath)

        elif 'send email' in query:
            try:
                speak("To whom you want to send the email")
                name = takeCommand().lower()
                speak("What is the subject")
                subject = takeCommand().capitalize()
                speak("What should i say?")
                content = takeCommand().capitalize()
                to = contact[name]
                sendEmail(to, subject, content)
                speak("Your email has been sent")

            except Exception as e:
                print(e)
                speak("sorry, i am enable to send the email.")

        elif 'quit' in query:
            exit()


"""
Exercicios do dia 48

Vamos usar o Selenium para automatizar alguns processos na internet.

"""

from selenium import webdriver
from selenium.webdriver.common.keys import Keys

CHROMIUM_DRIVER_PATH = r'/usr/lib/chromium/chromedriver'

driver = webdriver.Chrome(executable_path=CHROMIUM_DRIVER_PATH)

driver.maximize_window()
driver.get('https://pt.wikipedia.org/wiki/Wikip%C3%A9dia:P%C3%A1gina_principal')

pesquisa = driver.find_element_by_name('search')
pesquisa.send_keys('Python')
pesquisa.send_keys(Keys.ENTER)

driver.quit()

Nivel - Intermediario dia 15 a 58/dia 48/interação.py


"""Structure classes."""

import numpy as np
import rmsd
import math
import warnings
from scipy.spatial.distance import cdist
from collections import Counter, OrderedDict, defaultdict
from .base import StructureClass, query, StructureSet

class AtomStructure:
    """A structure made of atoms. This contains various useful methods that rely
    on a ``atoms()`` method, which the inheriting object must supply itself. All
    atomic structures also have IDs and names.

    Two atomic structures are equal if every pairwise atom in their pairing
    are equal.

    The class would never be instantiated directly."""

    def __init__(self, id=None, name=None):
        self._id, self._name = id, name

    def __eq__(self, other):
        try:
            mapping = self.pairing_with(other)
            for atom1, atom2 in mapping.items():
                if not atom1 == atom2: return False
            return True
        except: return False

    def __hash__(self):
        return id(self)

    @property
    def id(self):
        """The structure's unique ID.

        :rtype: ``str``"""

        return self._id

    @property
    def name(self):
        """The structure's name.

        :rtype: ``str``"""

        return self._name

    @name.setter
    def name(self, name):
        self._name = name

    @property
    def mass(self):
        """The structure's mass - the sum of all its atoms' masses.

        :rtype: ``float``"""

        return round(sum([atom.mass for atom in self.atoms()]), 12)

    @property
    def charge(self):
        """The structure's charge - the sum of all its atoms' charges.

        :rtype: ``float``"""

        return round(sum([atom.charge for atom in self.atoms()]), 12)

    @property
    def formula(self):
        """The structure's formula as a ``Counter`` dictionary - the count of
        all its atoms' elements.

        :rtype: ``Counter``"""

        return Counter([atom.element for atom in self.atoms()])

    @property
    def center_of_mass(self):
        """Returns the center of mass of the structure. This is the average of
        all the atom coordinates, weighted by the mass of each atom.

        :rtype: ``tuple``"""

        mass = self.mass
        locations = np.array([a._location * a.mass for a in self.atoms()])
        return np.sum(locations, axis=0) / mass

    @property
    def radius_of_gyration(self):
        """The radius of gyration of a structure is a measure of how extended it
        is. It is the root mean square deviation of the atoms' distance from the
        structure's :py:meth:`.center_of_mass`.

        :rtype: ``float``"""

        center_of_mass = self.center_of_mass
        atoms = self.atoms()
        square_deviation = sum(
         [atom.distance_to(center_of_mass) ** 2 for atom in atoms]
        )
        mean_square_deviation = square_deviation / len(atoms)
        return np.sqrt(mean_square_deviation)

    def pairing_with(self, structure):
        """Takes another structure with the same number of atoms as this one,
        and attempts to find the nearest equivalent of every atom in this
        structure, in that structure.

        Atoms will be aligned first by ID (if equal), then element, then by
        name, and finally by memory address - this last metric is
        used to ensure that even when allocation is essentially random, it is at
        least the same every time two structures are aligned.

        :param AtomStructure structure: the structure to pair with.
        :raises ValueError: if the other structure has a different number of\
        atoms.
        :rtype: ``dict``"""

        atoms = self.atoms()
        other_atoms = structure.atoms()
        if len(atoms) != len(other_atoms):
            raise ValueError("{} and {} have different numbers of atoms".format(
             self, structure
            ))
        pair = {}
        common_ids = set(a._id for a in atoms) & set(a._id for a in other_atoms)
        id_atoms = {a._id: a for a in atoms}
        id_other_atoms = {a._id: a for a in other_atoms}
        for id_ in common_ids:
            pair[id_atoms[id_]] = id_other_atoms[id_]
            atoms.remove(id_atoms[id_])
            other_atoms.remove(id_other_atoms[id_])
        atoms, other_atoms = list(atoms), list(other_atoms)
        for l in atoms, other_atoms:
            l.sort(key=lambda a: (
             a._id, a._element, a._name, id(a)
            ))
        return {**pair, **{a1: a2 for a1, a2 in zip(atoms, other_atoms)}}

    def rmsd_with(self, structure):
        """Calculates the Root Mean Square Deviation between this structure and
        another.

        :param AtomStructure structure: the structure to check against.
        :raises ValueError: if the other structure has a different number of\
        atoms.
        :rtype: ``float``"""

        pairing = self.pairing_with(structure)
        coords1, coords2 = [[a.location for a in atoms]
         for atoms in zip(*pairing.items())]
        c1, c2 = self.center_of_mass, structure.center_of_mass
        coords1 = [[x - c1[0], y - c1[1], z - c1[2]] for x, y, z in coords1]
        coords2 = [[x - c2[0], y - c2[1], z - c2[2]] for x, y, z in coords2]
        return round(rmsd.kabsch_rmsd(coords1, coords2), 12)

    def create_grid(self, size=1, margin=0):
        """A generator which models a grid around the structure and returns the
        coordinates of all the points in that grid. The origin is always one of
        those points, and the grid will be a box.

        :param int size: The spacing between grid points. The default is 1.
        :param int margin: How far to extend the grid beyond the structure\
        coordinates. The default is 0.
        :rtype: ``tuple``"""

        atom_locations = [atom.location for atom in self.atoms()]
        dimension_values = []
        for dimension in range(3):
            coordinates = [loc[dimension] for loc in atom_locations]
            min_, max_ = min(coordinates) - margin, max(coordinates) + margin
            values = [0]
            while values[0] > min_: values.insert(0, values[0] - size)
            while values[-1] < max_: values.append(values[-1] + size)
            dimension_values.append(values)
        for x in dimension_values[0]:
            for y in dimension_values[1]:
                for z in dimension_values[2]:
                    yield (x, y, z)

    def check_ids(self):
        """Looks through all the structure's sub-structures and raises a
        warning if they have duplicate ID."""

        for objects in ("chains", "ligands", "waters", "residues", "atoms"):
            try:
                ids = [obj.id for obj in getattr(self, objects)()]
                unique_ids = set(ids)
                if len(ids) != len(unique_ids):
                    warnings.warn(f"{objects} have duplicate IDs")
            except AttributeError: pass

    def save(self, path):
        """Saves the structure to file. The file extension given in the filename
        will be used to determine which file format to save in.

        If the structure you are saving has any duplicate IDs, a warning will be
        issued, as the file saved will likely be nonsensical.

        :param str path: the filename and location to save to."""

        from .utilities import save
        self.check_ids()
        ext = path.split(".")[-1]
        if ext == "cif":
            from .mmcif import structure_to_mmcif_string
            string = structure_to_mmcif_string(self)
        elif ext == "mmtf":
            from .mmtf import structure_to_mmtf_string
            string = structure_to_mmtf_string(self)
        elif ext == "pdb":
            from .pdb import structure_to_pdb_string
            string = structure_to_pdb_string(self)
        else:
            raise ValueError("Unsupported file extension: " + ext)
        save(string, path)

    def atoms_in_sphere(self, location, radius, *args, **kwargs):
        """Returns all the atoms in a given sphere within this structure. This
        will be a lot faster if the structure is a :py:class:`.Model` and if
        :py:meth:`.optimise_distances` has been called, as it won't have to
        search all atoms.

        :param tuple location: the centre of the sphere.
        :param float radius: the radius of the sphere.
        :rtype: ``set``"""

        if "_internal_grid" in self.__dict__ and self._internal_grid:
            r, atoms = math.ceil(radius / 10), set()
            x, y, z = [int(math.floor(n / 10)) * 10 for n in location]
            x_range, y_range, z_range = [
             [(val - (n * 10)) for n in range(1, r + 1)][::-1] + [val] + [
              (val + n * 10) for n in range(1, r + 1)
             ] for val in (x, y, z)
            ]
            for x in x_range:
                for y in y_range:
                    for z in z_range:
                        atoms = atoms.union(self._internal_grid[x][y][z])
            atoms = StructureSet(*atoms)
            atoms = query(lambda self: atoms)(self, *args, **kwargs)
        else:
            atoms = self.atoms(*args, **kwargs)
        X = np.tile(location, [len(atoms), 1])
        Y = np.array([a.location for a in atoms])
        distances = cdist(X, Y)[0]
        return {a for index, a in enumerate(atoms) if distances[index] <= radius}

    def pairwise_atoms(self, *args, **kwargs):
        """A generator which yeilds all the pairwise atom combinations of the
        structure. There will be no duplicates in the returned generator, and
        the number of returned pairs will be a triangle number.

        :rtype: ``tuple``"""

        atoms = list(self.atoms(*args, **kwargs))
        for a_index in range(len(atoms) - 1):
            for o_index in range(a_index + 1, len(atoms)):
                yield {atoms[a_index], atoms[o_index]}

    def nearby_atoms(self, *args, **kwargs):
        """Returns all atoms within a given distance of this structure,
        excluding the structure's own atoms.

        This will be a lot faster if the model's
        :py:meth:`.optimise_distances` has been called, as it won't have to
        search all atoms.

        :param float cutoff: the distance cutoff to use.
        :rtype: ``set``"""

        atoms = set()
        for atom in self.atoms():
            atoms.update(atom.nearby_atoms(*args, **kwargs))
        return atoms - self.atoms()

    def nearby_hets(self, *args, **kwargs):
        """Returns all other het structures within a given distance of this
        structure, excluding itself.

        This will be a lot faster if the model's
        :py:meth:`.optimise_distances` has been called, as it won't have to
        search all atoms.

        :param float cutoff: the distance cutoff to use.
        :rtype: ``set``"""

        structures = set()
        hets = set()
        for atom in self.atoms():
            structures.update(atom.nearby_hets(*args, **kwargs))
            hets.add(atom.het)
        return structures - hets

    def nearby_chains(self, *args, **kwargs):
        """Returns all other chain structures within a given distance of this
        structure, excluding itself.

        :param float cutoff: the distance cutoff to use.
        :rtype: ``set``"""

        structures = set()
        chains = set()
        for atom in self.atoms():
            structures.update(atom.nearby_chains(*args, **kwargs))
            chains.add(atom.chain)
        return structures - chains

    def translate(self, dx=0, dy=0, dz=0, trim=12):
        """Translates the structure through space, updating all atom
        coordinates accordingly. You can provide three values, or a single
        vector.

        :param Number dx: The distance to move in the x direction.
        :param Number dy: The distance to move in the y direction.
        :param Number dz: The distance to move in the z direction.
        :param int trim: The amount of rounding to do to the atoms' coordinates\
        after translating - the default is 12 decimal places but this can be\
        set to ``None`` if no rounding is to be done."""

        try:
            _,_,_ = dx
            vector = dx
        except TypeError: vector = (dx, dy, dz)
        Atom.translate_atoms(vector, *self.atoms())
        self.trim(trim)

    def transform(self, matrix, trim=12):
        """Transforms the structure using a 3x3 matrix supplied. This is useful
        if the :py:meth:`.rotate` method isn't powerful enough for your needs.

        :param array matrix: A NumPy matrix representing the transformation.\
        You can supply a list of lists if you like and it will be converted to\
        a NumPy matrix.
        :param int trim: The amount of rounding to do to the atoms' coordinates\
        after transforming - the default is 12 decimal places but this can be\
        set to ``None`` if no rounding is to be done."""

        Atom.transform_atoms(matrix, *self.atoms())
        self.trim(trim)

    def rotate(self, angle, axis, trim=12):
        """Rotates the structure about an axis, updating all atom coordinates
        accordingly.

        :param Number angle: The angle in radians.
        :param str axis: The axis to rotate around. Can only be 'x', 'y' or 'z'.
        :param int trim: The amount of rounding to do to the atoms' coordinates\
        after translating - the default is 12 decimal places but this can be\
        set to ``None`` if no rounding is to be done."""

        Atom.rotate_atoms(angle, axis, *self.atoms())
        self.trim(trim)

    def trim(self, places):
        """Rounds the coordinate values to a given number of decimal places.
        Useful for removing floating point rounding errors after transformation.

        :param int places: The number of places to round the coordinates to. If\
        ``None``, no rounding will be done."""

        for atom in self.atoms():
            atom.trim(places)

class Molecule(AtomStructure):
    """A molecule is a top-level constituent of a :py:class:`.Model` - a chain,
    a ligand, or a water molecule. They can have internal IDs, separate from the
    standard ID."""

    def __init__(self, id, name, internal_id):
        AtomStructure.__init__(self, id, name)
        self._internal_id = internal_id
        self._model = None

    @property
    def internal_id(self):
        """The molecule's internal ID - how it is refered to by atomium
        operations. This will be identical to regular IDs when the model comes
        from a .pdb file, but .cif and .mmtf files make this distinction.

        :rtype: ``str``"""

        return self._internal_id or self._id

    @property
    def model(self):
        """Returns the molecules :py:class:`.Model`.

        :rtype: ``Model``"""

        return self._model

class Het(AtomStructure):
    """A direct container of atoms, such as a residue or ligand. Though never
    instantiated directly, there is an initaliser method for setting up the
    atom dictionary."""

    from atomium import data as __data

    def __init__(self, id, name, full_name, *atoms):
        AtomStructure.__init__(self, id, name)
        self._full_name = full_name
        for atom in atoms: atom._het = self
        self._atoms = StructureSet(*atoms)

    def __contains__(self, atom):
        return atom in self._atoms.structures

    @property
    def full_name(self):
        """Returns the residue's full name, based on its three letter name - or
        just the three letter name if it doesn't match anything. Or you can just
        supply a full name when you instantiate the Het.

        :rtype: ``str``"""

        if self._full_name: return self._full_name
        return self.__data.FULL_NAMES.get(self._name, self._name)

    @full_name.setter
    def full_name(self, full_name):
        self._full_name = full_name

    @property
    def chain(self):
        """Returns the :py:class:`.Chain` the structure is part of (if a
        residue) or associated with (if a ligand).

        :rtype: ``Chain``"""

        return self._chain

    def atoms(self):
        """Returns the atoms in the ligand.

        :rtype: ``set``"""

        return self._atoms

class Model(AtomStructure, metaclass=StructureClass):
    """The universe in which all other molecules live, interact, and generally
    exist.

    It is a cotainer of its molecules, residues, and atoms.

    :param \*molecules: The chains, ligands, and waters that will inhabit the\
    model."""

    def __init__(self, *molecules, file=None):
        AtomStructure.__init__(self, None, None)
        self._chains = set()
        self._ligands = set()
        self._waters = set()
        for mol in molecules:
            mol._model = self
            d = (self._chains if isinstance(mol, Chain) else self._waters
             if mol._water else self._ligands)
            d.add(mol)
        self._chains = StructureSet(*self._chains)
        self._ligands = StructureSet(*self._ligands)
        self._waters = StructureSet(*self._waters)
        self._file = file
        self._internal_grid = None

    def __repr__(self):
        chains = "{} chains".format(len(self._chains))
        if len(self._chains) == 1: chains = chains[:-1]
        ligands = "{} ligands".format(len(self._ligands))
        if len(self._ligands) == 1: ligands = ligands[:-1]
        return "<Model ({}, {})>".format(chains, ligands)

    def __contains__(self, obj):
        return (obj in self.molecules() or obj in self.residues()
         or obj in self.atoms())

    @property
    def file(self):
        """The :py:class:`.File` the model comes from."""

        return self._file

    def chains(self):
        """Returns the model's chains.

        :rtype: ``set``"""

        return self._chains

    def ligands(self):
        """Returns the model's ligands.

        :rtype: ``set``"""

        return self._ligands

    def waters(self):
        """Returns the model's water ligands.

        :rtype: ``set``"""

        return self._waters

    def molecules(self):
        """Returns all of the model's molecules (chains, ligands, waters).

        :rtype: ``set``"""

        return self._chains + self._ligands + self._waters

    def residues(self):
        """Returns all of the model's residues in all its chains.

        :rtype: ``set``"""

        res = []
        for chain in self._chains.structures:
            res += chain.residues()
        return StructureSet(*res)

    def atoms(self):
        """Returns all of the model's atoms in all its molecules.

        :rtype: ``set``"""

        atoms = set()
        for mol in self.molecules():
            try:
                atoms.update(mol._atoms.structures)
            except:
                for res in mol._residues.structures:
                    atoms.update(res._atoms.structures)
        return StructureSet(*atoms)

    def dehydrate(self):
        """Removes all water ligands from the model."""

        self._waters = StructureSet()

    def optimise_distances(self):
        """Calling this method makes finding atoms within a sphere faster, and
        consequently makes all 'nearby' methods faster. It organises the atoms
        in the model into grids, so that only relevant atoms are checked for
        distances."""

        self._internal_grid = defaultdict(
         lambda: defaultdict(lambda: defaultdict(set))
        )
        for atom in self.atoms():
            x, y, z = [int(math.floor(n / 10)) * 10 for n in atom.location]
            self._internal_grid[x][y][z].add(atom)

    #TODO copy

class Chain(Molecule, metaclass=StructureClass):
    """A sequence of residues. Unlike other structures, they are iterable, and
    have a length.

    Residues can also be accessed using indexing.

    :param \*residues: The residues that will make up the chain.
    :param str id: the chain's unique ID.
    :param str internal_id: the internal ID used for transformations.
    :param str sequence: the actual sequence the chain should have.
    :param list helices: the alpha helices within the chain.
    :param list strands: the beta strands within the chain."""

    def __init__(self, *residues, sequence="", helices=None, strands=None, information=None, **kwargs):
        Molecule.__init__(
         self, kwargs.get("id"), kwargs.get("name"), kwargs.get("internal_id")
        )
        self._sequence = sequence
        for res in residues: res._chain = self
        self._residues = StructureSet(*residues)
        self._model = None
        self._helices = helices or []
        self._strands = strands or []
        self._information = information or {}

    def __repr__(self):
        return "<Chain {} ({} residues)>".format(self._id, len(self._residues))

    def __len__(self):
        return len(self._residues)

    def __iter__(self):
        return iter(self._residues.structures)

    def __getitem__(self, key):
        return self.residues()[key]

    def __contains__(self, obj):
        return obj in self._residues.structures or obj in self.atoms()

    @property
    def sequence(self):
        """Returns the sequence associated with the chain. Note that this is the
        sequence that the molecule actually has in real life - some may be
        missing from the actual sequence of residues in the structure.

        :rtype: ``str``"""

        return self._sequence

    @sequence.setter
    def sequence(self, sequence):
        self._sequence = sequence

    @property
    def helices(self):
        """The alpha helix residues in the chain

        :rtype: ``tuple``"""

        return tuple(self._helices)

    @property
    def strands(self):
        """The beta strand residues in the chain

        :rtype: ``tuple``"""

        return tuple(self._strands)

    @property
    def information(self):
        """The source organism and other information related to the chain

        :rtype: ``dict``"""

        return self._information

    @property
    def length(self):
        """Returns the number of residues in the chain.

        :rtype: ``int``"""

        return len(self)

    @property
    def present_sequence(self):
        """The sequence of residues actually present in the atoms present.

        :rtype: ``str``"""

        return "".join(r.code for r in self.residues())

    def copy(self, id=None, residue_ids=None, atom_ids=None):
        """Creates a copy of the chain, with new atoms and residues.

        :param str id: if given, the ID of the new chain.
        :param function residue_ids: a callable which, if given, will generate\
        new residue IDs.
        :param function atom_ids: a callable which, if given, will generate new\
        atom IDs.
        :rtype: ``Chain``"""

        residue_ids = residue_ids or (lambda i: i)
        residues = {r: r.copy(
         id=residue_ids(r.id), atom_ids=atom_ids
        ) for r in self.residues()}
        for r in self.residues():
            residues[r].next = residues[r.next] if r.next else None
        return Chain(
         *residues.values(), id=id or self._id, internal_id=self._internal_id,
         name=self._name, sequence=self._sequence,
         helices=[tuple(residues[r] for r in h) for h in self._helices],
         strands=[tuple(residues[r] for r in s) for s in self._strands]
        )

    def residues(self):
        """Returns the residues in the chain.

        :rtype: ``tuple``"""

        return self._residues

    def ligands(self):
        """Returns all the ligands associated with the chain - but only if the
        chain is part of a model.

        :rtype: ``set``"""

        return StructureSet() if self._model is None else StructureSet(
         *[l for l in self._model._ligands.structures if l._chain is self]
        )

    def atoms(self):
        """Returns all the atoms in with the chain.

        :rtype: ``set``"""

        atoms = set()
        for res in self._residues.structures:
            atoms.update(res._atoms.structures)
        return StructureSet(*atoms)

class Ligand(Molecule, Het, metaclass=StructureClass):
    """A small molecule, usually associated with a polymer chain.

    :param \*atoms: The atoms that will make up the ligand.
    :param str id: the ligand's unique ID.
    :param str name: the ligand's name.
    :param str internal_id: the internal ID used for transformations.
    :param Chain chain: the chain the ligand is associated with.
    :param bool water: if ``True``, the ligand will be treated as water."""

    def __init__(self, *atoms, chain=None, water=False, **kwargs):
        Het.__init__(
        self, kwargs.get("id"), kwargs.get("name"),
         kwargs.get("full_name"), *atoms)
        Molecule.__init__(self, kwargs.get("id"), kwargs.get("name"),
         kwargs.get("internal_id"))
        self._chain, self._water = chain, water

    def __repr__(self):
        return "<{} {} ({})>".format(
         "Water" if self._water else "Ligand", self._name, self._id
        )

    @property
    def is_water(self):
        """Returns ``True`` if the ligand is a water ligand.

        :rtype: ``bool``"""

        return self._water

    def copy(self, id=None, atom_ids=None):
        """Creates a copy of the ligand, with new atoms.

        :param str id: if given, the ID of the new ligand.
        :param function atom_ids: a callable which, if given, will generate new\
        atom IDs.
        :rtype: ``Ligand``"""

        atoms = list(self.atoms())
        if atom_ids:
            new_ids = [atom_ids(a.id) for a in atoms]
            atoms = [a.copy(id=id) for a, id in zip(atoms, new_ids)]
        else:
            atoms = [a.copy() for a in self.atoms()]
        return self.__class__(*atoms, id=id or self._id,
         name=self._name, internal_id=self._internal_id, water=self._water)

class Residue(Het, metaclass=StructureClass):
    """A small subunit within a chain.

    :param \*atoms: The atoms the residue is to be made of.
    :param str id: The residue's ID.
    :param str name: The residue's name."""

    from atomium import data as __data

    def __init__(self, *atoms, **kwargs):
        Het.__init__(self, kwargs.get("id"), kwargs.get("name"),
         kwargs.get("full_name"), *atoms)
        self._next, self._previous = None, None
        self._chain = None

    def __repr__(self):
        return "<Residue {} ({})>".format(self._name, self._id)

    @property
    def next(self):
        """Residues can be linked to each other in a linear chain. This property
        returns the :py:class:`.Residue` downstream of this one. Alternatively,
        if you supply a residue, that residue will be assigned as the 'next' one
        downstream to this, and this residue will be upstream to that.
        Note that is a separate concept from bonds.

        :raises ValueError: if you try to connect a residue to itself.
        :rtype: ``Residue``"""

        return self._next

    @next.setter
    def next(self, next):
        if next is None:
            if self._next: self._next._previous = None
            self._next = None
        elif next is self:
            raise ValueError("Cannot link {} to itself".format(self))
        else:
            self._next = next
            next._previous = self

    @property
    def previous(self):
        """Residues can be linked to each other in a linear chain. This property
        returns the :py:class:`.Residue` upstream of this one. Alternatively,
        if you supply a residue, that residue will be assigned as the 'previous'
        one upstream to this, and this residue will be downstream to that.

        :raises ValueError: if you try to connect a residue to itself.
        :rtype: ``Residue``"""

        return self._previous

    @previous.setter
    def previous(self, previous):
        if previous is None:
            if self._previous: self._previous._next = None
            self._previous = None
        elif previous is self:
            raise ValueError("Cannot link {} to itself".format(self))
        else:
            self._previous = previous
            previous._next = self

    @property
    def code(self):
        """Returns the single letter code, based on its three letter name - or
        just 'X' if it doesn't match anything.

        :rtype: ``str``"""

        return self.__data.CODES.get(self._name, "X")

    @property
    def helix(self):
        """Returns ``True`` if the residue is part of an alpha helix.

        :rtype: ``bool``"""

        if self.chain:
            for helix in self.chain.helices:
                if self in helix: return True
        return False

    @property
    def strand(self):
        """Returns ``True`` if the residue is part of a beta strand.

        :rtype: ``bool``"""

        if self.chain:
            for strand in self.chain.strands:
                if self in strand: return True
        return False

    def copy(self, id=None, atom_ids=None):
        """Creates a copy of the residue, with new atoms.

        :param str id: if given, the ID of the new residue.
        :param function atom_ids: a callable which, if given, will\
        generate new atom IDs.
        :rtype: ``Residue``"""

        atoms = list(self.atoms())
        if atom_ids:
            new_ids = [atom_ids(a.id) for a in atoms]
            atoms = [a.copy(id=id) for a, id in zip(atoms, new_ids)]
        else:
            atoms = [a.copy() for a in self.atoms()]
        return self.__class__(*atoms, id=id or self._id, name=self._name)

    @property
    def model(self):
        """Returns the :py:class:`.Model` the residue is part of, via its
        chain.

        :rtype: ``Model``"""

        try:
            return self._chain._model
        except AttributeError: return None

class Atom:
    """An atom in space - a point particle with a location, element, charge etc.

    Atoms are the building blocks of all structures in atomium.

    Two atoms are equal if they have the same properties (not including ID).

    :param str element: The atom's elemental symbol.
    :param number x: The atom's x coordinate.
    :param number y: The atom's y coordinate.
    :param number z: The atom's z coordinate.
    :param int id: An integer ID for the atom.
    :param str name: The atom's name.
    :param number charge: The charge of the atom.
    :param number bvalue: The B-value of the atom (its uncertainty).
    :param list anisotropy: The directional uncertainty of the atom."""

    from atomium import data as __data

    __slots__ = [
     "_element", "_location", "_id", "_name", "_charge",
     "_bvalue", "_anisotropy", "_het", "_bonded_atoms", "_is_hetatm"
    ]

    def __init__(self, element, x, y, z, id, name, charge, bvalue, anisotropy, is_hetatm=False):
        self._location = np.array([x, y, z])
        self._element = element
        self._id, self._name, self._charge = id, name, charge
        self._bvalue, self._anisotropy = bvalue, anisotropy
        self._het, self._bonded_atoms, self._is_hetatm = None, set(), is_hetatm

    def __repr__(self):
        return "<Atom {} ({})>".format(self._id, self._name)

    def __iter__(self):
        return iter(self._location)

    def __eq__(self, other):
        if not isinstance(other, Atom): return False
        for attr in self.__slots__:
            if attr not in ("_id", "_het", "_bonded_atoms", "_location"):
                if getattr(self, attr) != getattr(other, attr): return False
            if list(self._location) != list(other._location): return False
        return True

    def __hash__(self):
        return id(self)

    @staticmethod
    def translate_atoms(vector, *atoms):
        """Translates multiple atoms using some vector.

        :param vector: the three values representing the delta position.
        :param \*atoms: the atoms to translate."""

        for atom in atoms:
            atom._location += np.array(vector)

    @staticmethod
    def transform_atoms(matrix, *atoms):
        """Transforms multiple atoms using some matrix.

        :param matrix: the transformation matrix.
        :param \*atoms: the atoms to transform."""

        locations = [list(a) for a in atoms]
        output = np.dot(np.array(matrix), np.array(locations).transpose())
        for atom, location in zip(atoms, output.transpose()):
            atom._location = location

    @staticmethod
    def rotate_atoms(angle, axis, *atoms, **kwargs):
        """Rotates multiple atoms using an axis and an angle.

        :param float angle: the angle to rotate by in radians.
        :param str axis: the axis to rotate around (x, y, or z).
        :param \*atoms: the atoms to rotate."""

        try:
            axis = [1 if i == "xyz".index(axis) else 0 for i in range(3)]
        except ValueError:
            raise ValueError("'{}' is not a valid axis".format(axis))
        axis = np.asarray(axis)
        axis = axis / np.sqrt(np.dot(axis, axis))
        a = np.cos(angle / 2)
        b, c, d = -axis * np.sin(angle / 2)
        aa, bb, cc, dd = a * a, b * b, c * c, d * d
        bc, ad, ac, ab, bd, cd = b * c, a * d, a * c, a * b, b * d, c * d
        Atom.transform_atoms(np.array([
         [aa + bb - cc - dd, 2 * (bc + ad), 2 * (bd - ac)],
         [2 * (bc - ad), aa + cc - bb - dd, 2 * (cd + ab)],
         [2 * (bd + ac), 2 * (cd - ab), aa + dd - bb - cc]
        ]), *atoms, **kwargs)

    @property
    def element(self):
        """The atom's element symbol. This is used to calculate its mass using a
        Periodic Table.

        :rtype: ``str``"""

        return self._element

    @property
    def location(self):
        """The atom's location.

        :rtype: ``tuple``"""

        return tuple(self._location)

    @property
    def id(self):
        """The atom's unique integer ID. It cannot be updated - the ID the atom
        is created with is its ID forever.

        :rtype: ``int``"""

        return self._id

    @property
    def name(self):
        """The atom's name. This is often used to determine what 'kind' of atom
        it is.

        :rtype: ``str``"""

        return self._name

    @name.setter
    def name(self, name):
        self._name = name

    @property
    def charge(self):
        """The atom's charge - usually just zero, or 'neutral'.

        :rtype: ``float``"""

        return self._charge

    @charge.setter
    def charge(self, charge):
        self._charge = charge

    @property
    def bvalue(self):
        """The atom's B-value - the uncertainty in its position in all
        directions.

        :rtype: ``float``"""

        return self._bvalue

    @bvalue.setter
    def bvalue(self, bvalue):
        self._bvalue = bvalue

    @property
    def anisotropy(self):
        """The atom's directional uncertainty, represented by a list of six
        numbers.

        :rtype: ``list``"""

        return self._anisotropy

    @property
    def bonded_atoms(self):
        """Returns the atoms this atom is bonded to.

        :rtype: ``set```"""

        return self._bonded_atoms

    @property
    def mass(self):
        """The atom's molar mass according to the Periodic Table, based on the
        atom's :py:meth:`element`. If the element doesn't match any symbol on
        the Periodic Table, a mass of 0 will be returned.

        The element lookup is case-insensitive.

        :rtype: ``float``"""

        return self.__data.PERIODIC_TABLE.get(self._element.upper(), 0)

    @property
    def atomic_number(self):
        """The atom's nuclear charge or atomic number, based on the
        atom's :py:meth:`element`. If the element doesn't match any symbol on
        the Periodic Table, a mass of 0 will be returned.

        The element lookup is case-insensitive.

        :rtype: ``int``"""

        return self.__data.ATOMIC_NUMBER.get(self._element.upper(), 0)

    @property
    def covalent_radius(self):
        """The atom's covalent radius, based on the atom's :py:meth:`element`.
        If the element doesn't match any symbol on the Periodic Table, a radius
        of 0 will be returned.

        The element lookup is case-insensitive.

        :rtype: ``float``"""

        return self.__data.COVALENT_RADII.get(self._element.upper(), 0)

    @property
    def is_metal(self):
        """Checks whether the atom's element matches a metal element.

        The element lookup is case-insensitive.

        :rtype: ``bool``"""

        return self._element.upper() in self.__data.METALS

    @property
    def is_backbone(self):
        """Returns ``True`` if the atom has a backbone atom name.

        :rtype: ``bool``"""

        return isinstance(self._het, Residue) and \
         self._name in ["CA", "C", "N", "O"]

    @property
    def is_side_chain(self):
        """Returns ``True`` if the atom has a side chain atom name.

        :rtype: ``bool``"""

        return isinstance(self._het, Residue) and not self.is_backbone

    def distance_to(self, other):
        """Returns the distance (in whatever units the coordinates are defined
        in) between this atom and another. You can also give a (x, y, z) tuple
        instead of another atom if you so wish.

        :param Atom other: The other atom (or location tuple).
        :rtype: ``float``"""

        return np.linalg.norm(self._location - np.array(list(other)))

    def angle(self, atom1, atom2):
        """Gets the angle between two atom vectors with this atom as the origin.

        :param Atom atom1: The first atom.
        :param Atom atom2: Thne second atom."""

        vectors = [
         [v1 - v2 for v1, v2 in zip(atom.location, self.location)
        ] for atom in (atom1, atom2)]
        normalized = [np.linalg.norm(v) for v in vectors]
        if 0 in normalized: return 0
        vectors = [v / n for v, n in zip(vectors, normalized)]
        return np.arccos(np.clip(np.dot(vectors[0], vectors[1]), -1.0, 1.0))

    def copy(self, id=None):
        """Returns a copy of the atom. The new atom will have the same element,
        location, name, charge, ID, bvalue etc. as the original, but will not
        be part of any model or other molecule.

        :rtype: ``Atom``"""

        return Atom(
         self._element, *self._location, id or self._id, self._name,
         self._charge, self._bvalue, self._anisotropy
        )

    @property
    def het(self):
        """Returns the :py:class:`.Residue` or :py:class:`.Ligand` the atom is
        part of, or ``None`` if it is not part of one.

        :rtype: ``Het```"""

        return self._het

    @property
    def chain(self):
        """Returns the :py:class:`.Chain` the atom is part of, or ``None`` if
        it is not part of one.

        :rtype: ``Chain``"""

        if self._het: return self._het.chain

    @property
    def model(self):
        """Returns the :py:class:`.Model` the atom is part of, or ``None`` if
        it is not part of one.

        :rtype: ``Model``"""

        if self.chain: return self.chain.model

    def nearby_atoms(self, cutoff, *args, **kwargs):
        """Returns all atoms in the associated :py:class:`.Model` that are
        within a given distance (in the units of the atom coordinates) of this
        atom. If the atom is not part of a model, no atoms will be returned.

        :param float cutoff: The radius to search within.
        :rtype: ``set``"""

        if self.model:
            atoms =  self.model.atoms_in_sphere(
             self.location, cutoff, *args, **kwargs
            )
            try:
                atoms.remove(self)
            except: pass
            return atoms
        return set()

    def nearby_hets(self, *args, residues=True, ligands=True, **kwargs):
        """Returns all residues and ligands in the associated :py:class:`.Model`
        that are within a given distance (in the units of the atom coordinates)
        of this atom. If the atom is not part of a model, no residues will be
        returned.

        :param float cutoff: the distance cutoff to use.
        :param bool residues: if ``False``, residues will not be returned.
        :param bool ligands: if ``False``, ligands will not be returned.
        :rtype: ``set``"""

        atoms = self.nearby_atoms(*args, **kwargs)
        structures = set()
        for atom in atoms:
            if atom.het is not None: structures.add(atom.het)
        try:
            structures.remove(self.het)
        except: pass
        if not residues:
            structures = {s for s in structures if not isinstance(s, Residue)}
        if not ligands:
            structures = {s for s in structures if not (isinstance(s, Ligand))}
        return structures

    def nearby_chains(self, *args, **kwargs):
        """Returns all chain structures in the associated :py:class:`.Model`
        that are within a given distance (in the units of the atom coordinates)
        of this atom. If the atom is not part of a model, no chains will be
        returned.

        :param float cutoff: the distance cutoff to use.
        :rtype: ``set``"""

        atoms = self.nearby_atoms(*args, **kwargs)
        chains = set()
        for atom in atoms:
            if atom.chain is not None: chains.add(atom.chain)
        try:
            chains.remove(self.chain)
        except: pass
        return chains

    def translate(self, dx=0, dy=0, dz=0, trim=12):
        """Translates an atom in 3D space. You can provide three values, or a
        single vector.

        :param float dx: The distance to move in the x direction.
        :param float dy: The distance to move in the y direction.
        :param float dz: The distance to move in the z direction.
        :param int trim: The amount of rounding to do to the atom's coordinates\
        after translating - the default is 12 decimal places but this can be\
        set to ``None`` if no rounding is to be done."""

        try:
            _,_,_ = dx
            vector = dx
        except TypeError: vector = (dx, dy, dz)
        Atom.translate_atoms(vector, self)
        self.trim(trim)

    def transform(self, matrix, trim=12):
        """Transforms the atom using a 3x3 matrix supplied. This is useful if
        the :py:meth:`.rotate` method isn't powerful enough for your needs.

        :param array matrix: A NumPy matrix representing the transformation.\
        You can supply a list of lists if you like and it will be converted to\
        a NumPy matrix.
        :param int trim: The amount of rounding to do to the atom's coordinates\
        after transforming - the default is 12 decimal places but this can be\
        set to ``None`` if no rounding is to be done."""

        Atom.transform_atoms(matrix, self)
        self.trim(trim)

    def rotate(self, angle, axis, trim=12):
        """Rotates the atom by an angle in radians, around one of the the three
        axes.

        :param float angle: The angle to rotate by in radians.
        :param str axis: the axis to rotate around.
        :param int trim: The amount of rounding to do to the atom's coordinates\
        after rotating - the default is 12 decimal places but this can be\
        set to ``None`` if no rounding is to be done."""

        Atom.rotate_atoms(angle, axis, self)
        self.trim(trim)

    def move_to(self, x, y, z):
        """Moves the atom to the coordinates given.

        :param number x: The atom's new x coordinate.
        :param number y: The atom's new y coordinate.
        :param number z: The atom's new z coordinate."""

        self._location[0], self._location[1], self._location[2] = x, y, z

    def trim(self, places):
        """Rounds the coordinate values to a given number of decimal places.
        Useful for removing floating point rounding errors after transformation.

        :param int places: The number of places to round the coordinates to. If\
        ``None``, no rounding will be done."""

        if places is not None:
            self._location = np.round(self._location, places)

    def bond(self, other):
        """Bonds the atom to some other atom.

        :param Atom other: the other atom to bond to."""

        self._bonded_atoms.add(other)
        other._bonded_atoms.add(self)

{'save': 'utilities.save', 'structure_to_mmcif_string': 'mmcif.structure_to_mmcif_string', 'structure_to_mmtf_string': 'mmtf.structure_to_mmtf_string', 'structure_to_pdb_string': 'pdb.structure_to_pdb_string'}.__init__


#!/usr/bin/python3
"""
query Reddit API recursively for list of hots
"""
import requests

def recurse(subreddit, hot_list=[], after="", count=0):
    """ get recursively

    Args:
        subreddit(str): subreddit name
        hot_list(list): [] list to hold hot topics
        after(str): "" to hold the after parameter
        count(int): 0 of topics found
    """

    if subreddit is None or type(subreddit) is not str:
        print(None)
    url = "https://www.reddit.com/r/{}/hot.json".format(subreddit)
    headers = {'User-Agent': 'alx-devops-0x16 - maradeben'}
    params = {
        "after": after,
        "count": count,
        "limit": 100
    }

    resp = requests.get(url, headers=headers, params=params,
                        allow_redirects=False)
    if resp.status_code == 200:
        resp_json = resp.json().get('data')
        after = resp_json.get("after")
        count += resp_json.get("dist")
        children = resp_json.get('children')
        for child in children:
            hot_list.append(child.get('data').get('title'))

        if after is not None:
            return (hot_list)
        else:
            return recurse(subreddit, hot_list, after, count)
    else:
        return(None)


import json

lines = [l.strip() for l in open('08_input.txt')]

src_sum = sum(len(l) for l in lines)
s1_sum = sum(len(eval(l)) for l in lines)
s2_sum = sum(len(json.dumps(l)) for l in lines)

print('Star 1:', src_sum - s1_sum)
print('Star 2:', s2_sum - src_sum)


from setuptools import setup, find_namespace_packages

def _parse_requirements(file_path):
    with open(file_path, 'r') as stream:
        file = stream.readlines()
    return file

setup(
    name="exposurestats",
    version="1.1.2",
    description="Photo Statistics for Exposure Image Editor",
    classifiers=[
        "Programming Language :: Python :: 3.10",
    ],
    keywords="photography",
    url="https://github.com/luismavs/ExposureStats",
    author="Luís Seabra",
    author_email="luismavseabra@gmail.com",
    install_requires=['pandas','xmltodict', 'altair', 'streamlit', 'pyyaml', 'watchdog', 'tqdm'],
    license="MIT",
    python_requires=">=3.8",
    packages=["exposurestats"],
    zip_safe=False,
)


#!/usr/bin/env python3

import argparse
import pysam
import csv
import random
import gzip

parser = argparse.ArgumentParser(description='foo')
parser.add_argument('-f','--fragments',action='store')
parser.add_argument('-b','--barcodes',action='store')
parser.add_argument('-o','--output',action='store')
args = parser.parse_args()

def parse_barcodes(args):
  bcd = {}
  with open(args.barcodes) as f:
    barcodes = csv.reader(f)
    for line in barcodes:
      bcd[line[1]] = line[0]
  return(bcd)

def filter_bam(args,bcd):
  reads     = {}
  replicate = {}
  with gzip.open(args.fragments) as f:
    tbx = pysam.tabix_iterator(f,pysam.asBed())
    for line in tbx:
      if line.name in bcd:
        try:
          reads[bcd[line.name] + "_rep" + line.name.split("_")[-2]].append(str(line))
        except KeyError:
          reads[bcd[line.name] + "_rep" + line.name.split("_")[-2]]     = [str(line)]

  return(reads)

def main(args):
  barcodes  = parse_barcodes(args)
  reads_dic = filter_bam(args,barcodes)
  for key in reads_dic:
    with open(args.output + key + ".bed",'w') as f:
      f.write("\n".join(reads_dic[key]))

main(args)


from collections import deque

n = int(input())
board = [list(map(int, input())) for _ in range(n)]

# deque 정의
d = deque()
# 결과값 리스트
res = []

# 방향벡터 정의
dx = [1,0,-1,0]
dy = [0,1,0,-1]

for i in range(n):
    for j in range(n):
        # 집이 있는 곳이라면 큐에 추가
        if board[i][j] == 1:
            board[i][j] = 0
            d.append((i,j))
            # 단지수 1부터 카운트
            cnt = 1
            # bfs 수행
            while d:
                x, y = d.popleft()
                for i in range(4):
                    nx = x + dx[i]
                    ny = y + dy[i]
                    if 0<=nx<n and 0<=ny<n and board[nx][ny]:
                        cnt += 1
                        board[nx][ny] = 0
                        d.append((nx, ny))
            res.append(cnt)

print(len(res))
for x in sorted(res):
    print(x)

# dfs 풀이
def dfs(x, y):
    global cnt
    for i in range(4):
        nx = x + dx[i]
        ny = y + dy[i]
        if 0<=nx<n and 0<=ny<n and board[nx][ny]:
            board[nx][ny] = 0
            cnt += 1
            dfs(nx, ny)

for i in range(n):
    for j in range(n):
        # 집이 있는 곳이라면 dfs 실행
        if board[i][j] == 1:
            board[i][j] = 0
            cnt = 1
            dfs(i, j)
            res.append(cnt)

print(len(res))
for x in sorted(res):
    print(x)

단지 번호 붙이기.py