# Setup
Because we switch to `tf2`, instead of using `tf.slim`, we will just use `tf.keras` for the inception model weights.

```python
%matplotlib inline
import matplotlib.pyplot as plt
import matplotlib as mpl
from PIL import Image

from tqdm.notebook import tqdm
```

```python
mpl.rc('axes', labelsize=14)
mpl.rc('xtick', labelsize=12)
mpl.rc('ytick', labelsize=12)

DARK_READER = True
if DARK_READER:
    plt.rcParams.update({
        "lines.color": "white",
        "patch.edgecolor": "white",
        "text.color": "black",
        "axes.facecolor": "black",
        "axes.edgecolor": "lightgray",
        "axes.labelcolor": "white",
        "axes.titlecolor": "white",
        "xtick.color": "white",
        "ytick.color": "white",
        "grid.color": "lightgray",
        "figure.facecolor": "black",
        "figure.edgecolor": "black",
        "savefig.facecolor": "black",
        "savefig.edgecolor": "black",
    })
```

```python
import tensorflow as tf
import tensorflow.keras as keras
tf.__version__, keras.__version__
```

    ('2.5.0', '2.5.0')

```python
inception_model = keras.applications.inception_v3.InceptionV3()
inception_model, type(inception_model)
```

    (<tensorflow.python.keras.engine.functional.Functional at 0x7f52e0110ad0>,
     tensorflow.python.keras.engine.functional.Functional)

```python
isinstance(inception_model, keras.Model)
```

    True

```python
def non_dunder(obj, strict=False):
    if strict:
        prefix = "_"
    else:
        prefix = "__"
    return [s for s in dir(obj) if not s.startswith(prefix)]
```

```python
inception_model.trainable = False
inception_model.trainable
```

    False

```python
inception_model.trainable_weights, inception_model.trainable_variables
```

    ([], [])

```python
inception_model.layers[0].input.shape
```

    TensorShape([None, 299, 299, 3])

```python
import numpy as np
```

```python
from tensorflow.keras.applications.inception_v3 import preprocess_input
```
preprocess_input(
    np.random.randint(low=0, high=255, size=(2,2,3)))random_tensor = tf.constant(
    np.random.randint(low=0, high=255, size=(2,2,3)),
    dtype=tf.float32,
)
#random_tensor.dtype
preprocess_input(random_tensor)

```python
def inception(image):
    if isinstance(image, tf.Variable):
        #image.assign(preprocess_input(image.numpy()))
        pass
    elif isinstance(image, np.ndarray) and image.dtype == np.uint8:
        image = preprocess_input(image)

    if len(image.shape) == 3:
        image_batch = tf.expand_dims(image, axis=0)

    #if image.dtype is np.uint8:
    #    image = image.astype(np.float32)
    #if image.dtype is tf.uint8:
    #    image = tf.cast(image, tf.float32)
    #image = preprocess_input(image)
    return inception_model(image_batch)[0]
```

```python
import json
from pathlib import Path
from urllib.request import urlretrieve
```
help(urlretrieve)

```python
imagenet_json = Path("imagenet.json")
if not imagenet_json.exists():
    imagenet_json, _ = urlretrieve(
        'https://www.anishathalye.com/media/2017/07/25/imagenet.json',
        imagenet_json,
    )

with open(imagenet_json) as f:
    imagenet_labels = json.load(f)
imagenet_labels
```

    ['tench, Tinca tinca',
     'goldfish, Carassius auratus',
     'great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias',
     'tiger shark, Galeocerdo cuvieri',
     'hammerhead, hammerhead shark',
     'electric ray, crampfish, numbfish, torpedo',
     'stingray',
     'cock',
     'hen',
     'ostrich, Struthio camelus',
     'brambling, Fringilla montifringilla',
     'goldfinch, Carduelis carduelis',
     'house finch, linnet, Carpodacus mexicanus',
     'junco, snowbird',
     'indigo bunting, indigo finch, indigo bird, Passerina cyanea',
     'robin, American robin, Turdus migratorius',
     'bulbul',
     'jay',
     'magpie',
     'chickadee',
     'water ouzel, dipper',
     'kite',
     'bald eagle, American eagle, Haliaeetus leucocephalus',
     'vulture',
     'great grey owl, great gray owl, Strix nebulosa',
     'European fire salamander, Salamandra salamandra',
     'common newt, Triturus vulgaris',
     'eft',
     'spotted salamander, Ambystoma maculatum',
     'axolotl, mud puppy, Ambystoma mexicanum',
     'bullfrog, Rana catesbeiana',
     'tree frog, tree-frog',
     'tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui',
     'loggerhead, loggerhead turtle, Caretta caretta',
     'leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea',
     'mud turtle',
     'terrapin',
     'box turtle, box tortoise',
     'banded gecko',
     'common iguana, iguana, Iguana iguana',
     'American chameleon, anole, Anolis carolinensis',
     'whiptail, whiptail lizard',
     'agama',
     'frilled lizard, Chlamydosaurus kingi',
     'alligator lizard',
     'Gila monster, Heloderma suspectum',
     'green lizard, Lacerta viridis',
     'African chameleon, Chamaeleo chamaeleon',
     'Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis',
     'African crocodile, Nile crocodile, Crocodylus niloticus',
     'American alligator, Alligator mississipiensis',
     'triceratops',
     'thunder snake, worm snake, Carphophis amoenus',
     'ringneck snake, ring-necked snake, ring snake',
     'hognose snake, puff adder, sand viper',
     'green snake, grass snake',
     'king snake, kingsnake',
     'garter snake, grass snake',
     'water snake',
     'vine snake',
     'night snake, Hypsiglena torquata',
     'boa constrictor, Constrictor constrictor',
     'rock python, rock snake, Python sebae',
     'Indian cobra, Naja naja',
     'green mamba',
     'sea snake',
     'horned viper, cerastes, sand viper, horned asp, Cerastes cornutus',
     'diamondback, diamondback rattlesnake, Crotalus adamanteus',
     'sidewinder, horned rattlesnake, Crotalus cerastes',
     'trilobite',
     'harvestman, daddy longlegs, Phalangium opilio',
     'scorpion',
     'black and gold garden spider, Argiope aurantia',
     'barn spider, Araneus cavaticus',
     'garden spider, Aranea diademata',
     'black widow, Latrodectus mactans',
     'tarantula',
     'wolf spider, hunting spider',
     'tick',
     'centipede',
     'black grouse',
     'ptarmigan',
     'ruffed grouse, partridge, Bonasa umbellus',
     'prairie chicken, prairie grouse, prairie fowl',
     'peacock',
     'quail',
     'partridge',
     'African grey, African gray, Psittacus erithacus',
     'macaw',
     'sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita',
     'lorikeet',
     'coucal',
     'bee eater',
     'hornbill',
     'hummingbird',
     'jacamar',
     'toucan',
     'drake',
     'red-breasted merganser, Mergus serrator',
     'goose',
     'black swan, Cygnus atratus',
     'tusker',
     'echidna, spiny anteater, anteater',
     'platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus',
     'wallaby, brush kangaroo',
     'koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus',
     'wombat',
     'jellyfish',
     'sea anemone, anemone',
     'brain coral',
     'flatworm, platyhelminth',
     'nematode, nematode worm, roundworm',
     'conch',
     'snail',
     'slug',
     'sea slug, nudibranch',
     'chiton, coat-of-mail shell, sea cradle, polyplacophore',
     'chambered nautilus, pearly nautilus, nautilus',
     'Dungeness crab, Cancer magister',
     'rock crab, Cancer irroratus',
     'fiddler crab',
     'king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica',
     'American lobster, Northern lobster, Maine lobster, Homarus americanus',
     'spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish',
     'crayfish, crawfish, crawdad, crawdaddy',
     'hermit crab',
     'isopod',
     'white stork, Ciconia ciconia',
     'black stork, Ciconia nigra',
     'spoonbill',
     'flamingo',
     'little blue heron, Egretta caerulea',
     'American egret, great white heron, Egretta albus',
     'bittern',
     'crane',
     'limpkin, Aramus pictus',
     'European gallinule, Porphyrio porphyrio',
     'American coot, marsh hen, mud hen, water hen, Fulica americana',
     'bustard',
     'ruddy turnstone, Arenaria interpres',
     'red-backed sandpiper, dunlin, Erolia alpina',
     'redshank, Tringa totanus',
     'dowitcher',
     'oystercatcher, oyster catcher',
     'pelican',
     'king penguin, Aptenodytes patagonica',
     'albatross, mollymawk',
     'grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus',
     'killer whale, killer, orca, grampus, sea wolf, Orcinus orca',
     'dugong, Dugong dugon',
     'sea lion',
     'Chihuahua',
     'Japanese spaniel',
     'Maltese dog, Maltese terrier, Maltese',
     'Pekinese, Pekingese, Peke',
     'Shih-Tzu',
     'Blenheim spaniel',
     'papillon',
     'toy terrier',
     'Rhodesian ridgeback',
     'Afghan hound, Afghan',
     'basset, basset hound',
     'beagle',
     'bloodhound, sleuthhound',
     'bluetick',
     'black-and-tan coonhound',
     'Walker hound, Walker foxhound',
     'English foxhound',
     'redbone',
     'borzoi, Russian wolfhound',
     'Irish wolfhound',
     'Italian greyhound',
     'whippet',
     'Ibizan hound, Ibizan Podenco',
     'Norwegian elkhound, elkhound',
     'otterhound, otter hound',
     'Saluki, gazelle hound',
     'Scottish deerhound, deerhound',
     'Weimaraner',
     'Staffordshire bullterrier, Staffordshire bull terrier',
     'American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier',
     'Bedlington terrier',
     'Border terrier',
     'Kerry blue terrier',
     'Irish terrier',
     'Norfolk terrier',
     'Norwich terrier',
     'Yorkshire terrier',
     'wire-haired fox terrier',
     'Lakeland terrier',
     'Sealyham terrier, Sealyham',
     'Airedale, Airedale terrier',
     'cairn, cairn terrier',
     'Australian terrier',
     'Dandie Dinmont, Dandie Dinmont terrier',
     'Boston bull, Boston terrier',
     'miniature schnauzer',
     'giant schnauzer',
     'standard schnauzer',
     'Scotch terrier, Scottish terrier, Scottie',
     'Tibetan terrier, chrysanthemum dog',
     'silky terrier, Sydney silky',
     'soft-coated wheaten terrier',
     'West Highland white terrier',
     'Lhasa, Lhasa apso',
     'flat-coated retriever',
     'curly-coated retriever',
     'golden retriever',
     'Labrador retriever',
     'Chesapeake Bay retriever',
     'German short-haired pointer',
     'vizsla, Hungarian pointer',
     'English setter',
     'Irish setter, red setter',
     'Gordon setter',
     'Brittany spaniel',
     'clumber, clumber spaniel',
     'English springer, English springer spaniel',
     'Welsh springer spaniel',
     'cocker spaniel, English cocker spaniel, cocker',
     'Sussex spaniel',
     'Irish water spaniel',
     'kuvasz',
     'schipperke',
     'groenendael',
     'malinois',
     'briard',
     'kelpie',
     'komondor',
     'Old English sheepdog, bobtail',
     'Shetland sheepdog, Shetland sheep dog, Shetland',
     'collie',
     'Border collie',
     'Bouvier des Flandres, Bouviers des Flandres',
     'Rottweiler',
     'German shepherd, German shepherd dog, German police dog, alsatian',
     'Doberman, Doberman pinscher',
     'miniature pinscher',
     'Greater Swiss Mountain dog',
     'Bernese mountain dog',
     'Appenzeller',
     'EntleBucher',
     'boxer',
     'bull mastiff',
     'Tibetan mastiff',
     'French bulldog',
     'Great Dane',
     'Saint Bernard, St Bernard',
     'Eskimo dog, husky',
     'malamute, malemute, Alaskan malamute',
     'Siberian husky',
     'dalmatian, coach dog, carriage dog',
     'affenpinscher, monkey pinscher, monkey dog',
     'basenji',
     'pug, pug-dog',
     'Leonberg',
     'Newfoundland, Newfoundland dog',
     'Great Pyrenees',
     'Samoyed, Samoyede',
     'Pomeranian',
     'chow, chow chow',
     'keeshond',
     'Brabancon griffon',
     'Pembroke, Pembroke Welsh corgi',
     'Cardigan, Cardigan Welsh corgi',
     'toy poodle',
     'miniature poodle',
     'standard poodle',
     'Mexican hairless',
     'timber wolf, grey wolf, gray wolf, Canis lupus',
     'white wolf, Arctic wolf, Canis lupus tundrarum',
     'red wolf, maned wolf, Canis rufus, Canis niger',
     'coyote, prairie wolf, brush wolf, Canis latrans',
     'dingo, warrigal, warragal, Canis dingo',
     'dhole, Cuon alpinus',
     'African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus',
     'hyena, hyaena',
     'red fox, Vulpes vulpes',
     'kit fox, Vulpes macrotis',
     'Arctic fox, white fox, Alopex lagopus',
     'grey fox, gray fox, Urocyon cinereoargenteus',
     'tabby, tabby cat',
     'tiger cat',
     'Persian cat',
     'Siamese cat, Siamese',
     'Egyptian cat',
     'cougar, puma, catamount, mountain lion, painter, panther, Felis concolor',
     'lynx, catamount',
     'leopard, Panthera pardus',
     'snow leopard, ounce, Panthera uncia',
     'jaguar, panther, Panthera onca, Felis onca',
     'lion, king of beasts, Panthera leo',
     'tiger, Panthera tigris',
     'cheetah, chetah, Acinonyx jubatus',
     'brown bear, bruin, Ursus arctos',
     'American black bear, black bear, Ursus americanus, Euarctos americanus',
     'ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus',
     'sloth bear, Melursus ursinus, Ursus ursinus',
     'mongoose',
     'meerkat, mierkat',
     'tiger beetle',
     'ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle',
     'ground beetle, carabid beetle',
     'long-horned beetle, longicorn, longicorn beetle',
     'leaf beetle, chrysomelid',
     'dung beetle',
     'rhinoceros beetle',
     'weevil',
     'fly',
     'bee',
     'ant, emmet, pismire',
     'grasshopper, hopper',
     'cricket',
     'walking stick, walkingstick, stick insect',
     'cockroach, roach',
     'mantis, mantid',
     'cicada, cicala',
     'leafhopper',
     'lacewing, lacewing fly',
     "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk",
     'damselfly',
     'admiral',
     'ringlet, ringlet butterfly',
     'monarch, monarch butterfly, milkweed butterfly, Danaus plexippus',
     'cabbage butterfly',
     'sulphur butterfly, sulfur butterfly',
     'lycaenid, lycaenid butterfly',
     'starfish, sea star',
     'sea urchin',
     'sea cucumber, holothurian',
     'wood rabbit, cottontail, cottontail rabbit',
     'hare',
     'Angora, Angora rabbit',
     'hamster',
     'porcupine, hedgehog',
     'fox squirrel, eastern fox squirrel, Sciurus niger',
     'marmot',
     'beaver',
     'guinea pig, Cavia cobaya',
     'sorrel',
     'zebra',
     'hog, pig, grunter, squealer, Sus scrofa',
     'wild boar, boar, Sus scrofa',
     'warthog',
     'hippopotamus, hippo, river horse, Hippopotamus amphibius',
     'ox',
     'water buffalo, water ox, Asiatic buffalo, Bubalus bubalis',
     'bison',
     'ram, tup',
     'bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis',
     'ibex, Capra ibex',
     'hartebeest',
     'impala, Aepyceros melampus',
     'gazelle',
     'Arabian camel, dromedary, Camelus dromedarius',
     'llama',
     'weasel',
     'mink',
     'polecat, fitch, foulmart, foumart, Mustela putorius',
     'black-footed ferret, ferret, Mustela nigripes',
     'otter',
     'skunk, polecat, wood pussy',
     'badger',
     'armadillo',
     'three-toed sloth, ai, Bradypus tridactylus',
     'orangutan, orang, orangutang, Pongo pygmaeus',
     'gorilla, Gorilla gorilla',
     'chimpanzee, chimp, Pan troglodytes',
     'gibbon, Hylobates lar',
     'siamang, Hylobates syndactylus, Symphalangus syndactylus',
     'guenon, guenon monkey',
     'patas, hussar monkey, Erythrocebus patas',
     'baboon',
     'macaque',
     'langur',
     'colobus, colobus monkey',
     'proboscis monkey, Nasalis larvatus',
     'marmoset',
     'capuchin, ringtail, Cebus capucinus',
     'howler monkey, howler',
     'titi, titi monkey',
     'spider monkey, Ateles geoffroyi',
     'squirrel monkey, Saimiri sciureus',
     'Madagascar cat, ring-tailed lemur, Lemur catta',
     'indri, indris, Indri indri, Indri brevicaudatus',
     'Indian elephant, Elephas maximus',
     'African elephant, Loxodonta africana',
     'lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens',
     'giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca',
     'barracouta, snoek',
     'eel',
     'coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch',
     'rock beauty, Holocanthus tricolor',
     'anemone fish',
     'sturgeon',
     'gar, garfish, garpike, billfish, Lepisosteus osseus',
     'lionfish',
     'puffer, pufferfish, blowfish, globefish',
     'abacus',
     'abaya',
     "academic gown, academic robe, judge's robe",
     'accordion, piano accordion, squeeze box',
     'acoustic guitar',
     'aircraft carrier, carrier, flattop, attack aircraft carrier',
     'airliner',
     'airship, dirigible',
     'altar',
     'ambulance',
     'amphibian, amphibious vehicle',
     'analog clock',
     'apiary, bee house',
     'apron',
     'ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin',
     'assault rifle, assault gun',
     'backpack, back pack, knapsack, packsack, rucksack, haversack',
     'bakery, bakeshop, bakehouse',
     'balance beam, beam',
     'balloon',
     'ballpoint, ballpoint pen, ballpen, Biro',
     'Band Aid',
     'banjo',
     'bannister, banister, balustrade, balusters, handrail',
     'barbell',
     'barber chair',
     'barbershop',
     'barn',
     'barometer',
     'barrel, cask',
     'barrow, garden cart, lawn cart, wheelbarrow',
     'baseball',
     'basketball',
     'bassinet',
     'bassoon',
     'bathing cap, swimming cap',
     'bath towel',
     'bathtub, bathing tub, bath, tub',
     'beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon',
     'beacon, lighthouse, beacon light, pharos',
     'beaker',
     'bearskin, busby, shako',
     'beer bottle',
     'beer glass',
     'bell cote, bell cot',
     'bib',
     'bicycle-built-for-two, tandem bicycle, tandem',
     'bikini, two-piece',
     'binder, ring-binder',
     'binoculars, field glasses, opera glasses',
     'birdhouse',
     'boathouse',
     'bobsled, bobsleigh, bob',
     'bolo tie, bolo, bola tie, bola',
     'bonnet, poke bonnet',
     'bookcase',
     'bookshop, bookstore, bookstall',
     'bottlecap',
     'bow',
     'bow tie, bow-tie, bowtie',
     'brass, memorial tablet, plaque',
     'brassiere, bra, bandeau',
     'breakwater, groin, groyne, mole, bulwark, seawall, jetty',
     'breastplate, aegis, egis',
     'broom',
     'bucket, pail',
     'buckle',
     'bulletproof vest',
     'bullet train, bullet',
     'butcher shop, meat market',
     'cab, hack, taxi, taxicab',
     'caldron, cauldron',
     'candle, taper, wax light',
     'cannon',
     'canoe',
     'can opener, tin opener',
     'cardigan',
     'car mirror',
     'carousel, carrousel, merry-go-round, roundabout, whirligig',
     "carpenter's kit, tool kit",
     'carton',
     'car wheel',
     'cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM',
     'cassette',
     'cassette player',
     'castle',
     'catamaran',
     'CD player',
     'cello, violoncello',
     'cellular telephone, cellular phone, cellphone, cell, mobile phone',
     'chain',
     'chainlink fence',
     'chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour',
     'chain saw, chainsaw',
     'chest',
     'chiffonier, commode',
     'chime, bell, gong',
     'china cabinet, china closet',
     'Christmas stocking',
     'church, church building',
     'cinema, movie theater, movie theatre, movie house, picture palace',
     'cleaver, meat cleaver, chopper',
     'cliff dwelling',
     'cloak',
     'clog, geta, patten, sabot',
     'cocktail shaker',
     'coffee mug',
     'coffeepot',
     'coil, spiral, volute, whorl, helix',
     'combination lock',
     'computer keyboard, keypad',
     'confectionery, confectionary, candy store',
     'container ship, containership, container vessel',
     'convertible',
     'corkscrew, bottle screw',
     'cornet, horn, trumpet, trump',
     'cowboy boot',
     'cowboy hat, ten-gallon hat',
     'cradle',
     'crane',
     'crash helmet',
     'crate',
     'crib, cot',
     'Crock Pot',
     'croquet ball',
     'crutch',
     'cuirass',
     'dam, dike, dyke',
     'desk',
     'desktop computer',
     'dial telephone, dial phone',
     'diaper, nappy, napkin',
     'digital clock',
     'digital watch',
     'dining table, board',
     'dishrag, dishcloth',
     'dishwasher, dish washer, dishwashing machine',
     'disk brake, disc brake',
     'dock, dockage, docking facility',
     'dogsled, dog sled, dog sleigh',
     'dome',
     'doormat, welcome mat',
     'drilling platform, offshore rig',
     'drum, membranophone, tympan',
     'drumstick',
     'dumbbell',
     'Dutch oven',
     'electric fan, blower',
     'electric guitar',
     'electric locomotive',
     'entertainment center',
     'envelope',
     'espresso maker',
     'face powder',
     'feather boa, boa',
     'file, file cabinet, filing cabinet',
     'fireboat',
     'fire engine, fire truck',
     'fire screen, fireguard',
     'flagpole, flagstaff',
     'flute, transverse flute',
     'folding chair',
     'football helmet',
     'forklift',
     'fountain',
     'fountain pen',
     'four-poster',
     'freight car',
     'French horn, horn',
     'frying pan, frypan, skillet',
     'fur coat',
     'garbage truck, dustcart',
     'gasmask, respirator, gas helmet',
     'gas pump, gasoline pump, petrol pump, island dispenser',
     'goblet',
     'go-kart',
     'golf ball',
     'golfcart, golf cart',
     'gondola',
     'gong, tam-tam',
     'gown',
     'grand piano, grand',
     'greenhouse, nursery, glasshouse',
     'grille, radiator grille',
     'grocery store, grocery, food market, market',
     'guillotine',
     'hair slide',
     'hair spray',
     'half track',
     'hammer',
     'hamper',
     'hand blower, blow dryer, blow drier, hair dryer, hair drier',
     'hand-held computer, hand-held microcomputer',
     'handkerchief, hankie, hanky, hankey',
     'hard disc, hard disk, fixed disk',
     'harmonica, mouth organ, harp, mouth harp',
     'harp',
     'harvester, reaper',
     'hatchet',
     'holster',
     'home theater, home theatre',
     'honeycomb',
     'hook, claw',
     'hoopskirt, crinoline',
     'horizontal bar, high bar',
     'horse cart, horse-cart',
     'hourglass',
     'iPod',
     'iron, smoothing iron',
     "jack-o'-lantern",
     'jean, blue jean, denim',
     'jeep, landrover',
     'jersey, T-shirt, tee shirt',
     'jigsaw puzzle',
     'jinrikisha, ricksha, rickshaw',
     'joystick',
     'kimono',
     'knee pad',
     'knot',
     'lab coat, laboratory coat',
     'ladle',
     'lampshade, lamp shade',
     'laptop, laptop computer',
     'lawn mower, mower',
     'lens cap, lens cover',
     'letter opener, paper knife, paperknife',
     'library',
     'lifeboat',
     'lighter, light, igniter, ignitor',
     'limousine, limo',
     'liner, ocean liner',
     'lipstick, lip rouge',
     'Loafer',
     'lotion',
     'loudspeaker, speaker, speaker unit, loudspeaker system, speaker system',
     "loupe, jeweler's loupe",
     'lumbermill, sawmill',
     'magnetic compass',
     'mailbag, postbag',
     'mailbox, letter box',
     'maillot',
     'maillot, tank suit',
     'manhole cover',
     'maraca',
     'marimba, xylophone',
     'mask',
     'matchstick',
     'maypole',
     'maze, labyrinth',
     'measuring cup',
     'medicine chest, medicine cabinet',
     'megalith, megalithic structure',
     'microphone, mike',
     'microwave, microwave oven',
     'military uniform',
     'milk can',
     'minibus',
     'miniskirt, mini',
     'minivan',
     'missile',
     'mitten',
     'mixing bowl',
     'mobile home, manufactured home',
     'Model T',
     'modem',
     'monastery',
     'monitor',
     'moped',
     'mortar',
     'mortarboard',
     'mosque',
     'mosquito net',
     'motor scooter, scooter',
     'mountain bike, all-terrain bike, off-roader',
     'mountain tent',
     'mouse, computer mouse',
     'mousetrap',
     'moving van',
     'muzzle',
     'nail',
     'neck brace',
     'necklace',
     'nipple',
     'notebook, notebook computer',
     'obelisk',
     'oboe, hautboy, hautbois',
     'ocarina, sweet potato',
     'odometer, hodometer, mileometer, milometer',
     'oil filter',
     'organ, pipe organ',
     'oscilloscope, scope, cathode-ray oscilloscope, CRO',
     'overskirt',
     'oxcart',
     'oxygen mask',
     'packet',
     'paddle, boat paddle',
     'paddlewheel, paddle wheel',
     'padlock',
     'paintbrush',
     "pajama, pyjama, pj's, jammies",
     'palace',
     'panpipe, pandean pipe, syrinx',
     'paper towel',
     'parachute, chute',
     'parallel bars, bars',
     'park bench',
     'parking meter',
     'passenger car, coach, carriage',
     'patio, terrace',
     'pay-phone, pay-station',
     'pedestal, plinth, footstall',
     'pencil box, pencil case',
     'pencil sharpener',
     'perfume, essence',
     'Petri dish',
     'photocopier',
     'pick, plectrum, plectron',
     'pickelhaube',
     'picket fence, paling',
     'pickup, pickup truck',
     'pier',
     'piggy bank, penny bank',
     'pill bottle',
     'pillow',
     'ping-pong ball',
     'pinwheel',
     'pirate, pirate ship',
     'pitcher, ewer',
     "plane, carpenter's plane, woodworking plane",
     'planetarium',
     'plastic bag',
     'plate rack',
     'plow, plough',
     "plunger, plumber's helper",
     'Polaroid camera, Polaroid Land camera',
     'pole',
     'police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria',
     'poncho',
     'pool table, billiard table, snooker table',
     'pop bottle, soda bottle',
     'pot, flowerpot',
     "potter's wheel",
     'power drill',
     'prayer rug, prayer mat',
     'printer',
     'prison, prison house',
     'projectile, missile',
     'projector',
     'puck, hockey puck',
     'punching bag, punch bag, punching ball, punchball',
     'purse',
     'quill, quill pen',
     'quilt, comforter, comfort, puff',
     'racer, race car, racing car',
     'racket, racquet',
     'radiator',
     'radio, wireless',
     'radio telescope, radio reflector',
     'rain barrel',
     'recreational vehicle, RV, R.V.',
     'reel',
     'reflex camera',
     'refrigerator, icebox',
     'remote control, remote',
     'restaurant, eating house, eating place, eatery',
     'revolver, six-gun, six-shooter',
     'rifle',
     'rocking chair, rocker',
     'rotisserie',
     'rubber eraser, rubber, pencil eraser',
     'rugby ball',
     'rule, ruler',
     'running shoe',
     'safe',
     'safety pin',
     'saltshaker, salt shaker',
     'sandal',
     'sarong',
     'sax, saxophone',
     'scabbard',
     'scale, weighing machine',
     'school bus',
     'schooner',
     'scoreboard',
     'screen, CRT screen',
     'screw',
     'screwdriver',
     'seat belt, seatbelt',
     'sewing machine',
     'shield, buckler',
     'shoe shop, shoe-shop, shoe store',
     'shoji',
     'shopping basket',
     'shopping cart',
     'shovel',
     'shower cap',
     'shower curtain',
     'ski',
     'ski mask',
     'sleeping bag',
     'slide rule, slipstick',
     'sliding door',
     'slot, one-armed bandit',
     'snorkel',
     'snowmobile',
     'snowplow, snowplough',
     'soap dispenser',
     'soccer ball',
     'sock',
     'solar dish, solar collector, solar furnace',
     'sombrero',
     'soup bowl',
     'space bar',
     'space heater',
     'space shuttle',
     'spatula',
     'speedboat',
     "spider web, spider's web",
     'spindle',
     'sports car, sport car',
     'spotlight, spot',
     'stage',
     'steam locomotive',
     'steel arch bridge',
     'steel drum',
     'stethoscope',
     'stole',
     'stone wall',
     'stopwatch, stop watch',
     'stove',
     'strainer',
     'streetcar, tram, tramcar, trolley, trolley car',
     'stretcher',
     'studio couch, day bed',
     'stupa, tope',
     'submarine, pigboat, sub, U-boat',
     'suit, suit of clothes',
     'sundial',
     'sunglass',
     'sunglasses, dark glasses, shades',
     'sunscreen, sunblock, sun blocker',
     'suspension bridge',
     'swab, swob, mop',
     'sweatshirt',
     'swimming trunks, bathing trunks',
     'swing',
     'switch, electric switch, electrical switch',
     'syringe',
     'table lamp',
     'tank, army tank, armored combat vehicle, armoured combat vehicle',
     'tape player',
     'teapot',
     'teddy, teddy bear',
     'television, television system',
     'tennis ball',
     'thatch, thatched roof',
     'theater curtain, theatre curtain',
     'thimble',
     'thresher, thrasher, threshing machine',
     'throne',
     'tile roof',
     'toaster',
     'tobacco shop, tobacconist shop, tobacconist',
     'toilet seat',
     'torch',
     'totem pole',
     'tow truck, tow car, wrecker',
     'toyshop',
     'tractor',
     'trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi',
     'tray',
     'trench coat',
     'tricycle, trike, velocipede',
     'trimaran',
     'tripod',
     'triumphal arch',
     'trolleybus, trolley coach, trackless trolley',
     'trombone',
     'tub, vat',
     'turnstile',
     'typewriter keyboard',
     'umbrella',
     'unicycle, monocycle',
     'upright, upright piano',
     'vacuum, vacuum cleaner',
     'vase',
     'vault',
     'velvet',
     'vending machine',
     'vestment',
     'viaduct',
     'violin, fiddle',
     'volleyball',
     'waffle iron',
     'wall clock',
     'wallet, billfold, notecase, pocketbook',
     'wardrobe, closet, press',
     'warplane, military plane',
     'washbasin, handbasin, washbowl, lavabo, wash-hand basin',
     'washer, automatic washer, washing machine',
     'water bottle',
     'water jug',
     'water tower',
     'whiskey jug',
     'whistle',
     'wig',
     'window screen',
     'window shade',
     'Windsor tie',
     'wine bottle',
     'wing',
     'wok',
     'wooden spoon',
     'wool, woolen, woollen',
     'worm fence, snake fence, snake-rail fence, Virginia fence',
     'wreck',
     'yawl',
     'yurt',
     'web site, website, internet site, site',
     'comic book',
     'crossword puzzle, crossword',
     'street sign',
     'traffic light, traffic signal, stoplight',
     'book jacket, dust cover, dust jacket, dust wrapper',
     'menu',
     'plate',
     'guacamole',
     'consomme',
     'hot pot, hotpot',
     'trifle',
     'ice cream, icecream',
     'ice lolly, lolly, lollipop, popsicle',
     'French loaf',
     'bagel, beigel',
     'pretzel',
     'cheeseburger',
     'hotdog, hot dog, red hot',
     'mashed potato',
     'head cabbage',
     'broccoli',
     'cauliflower',
     'zucchini, courgette',
     'spaghetti squash',
     'acorn squash',
     'butternut squash',
     'cucumber, cuke',
     'artichoke, globe artichoke',
     'bell pepper',
     'cardoon',
     'mushroom',
     'Granny Smith',
     'strawberry',
     'orange',
     'lemon',
     'fig',
     'pineapple, ananas',
     'banana',
     'jackfruit, jak, jack',
     'custard apple',
     'pomegranate',
     'hay',
     'carbonara',
     'chocolate sauce, chocolate syrup',
     'dough',
     'meat loaf, meatloaf',
     'pizza, pizza pie',
     'potpie',
     'burrito',
     'red wine',
     'espresso',
     'cup',
     'eggnog',
     'alp',
     'bubble',
     'cliff, drop, drop-off',
     'coral reef',
     'geyser',
     'lakeside, lakeshore',
     'promontory, headland, head, foreland',
     'sandbar, sand bar',
     'seashore, coast, seacoast, sea-coast',
     'valley, vale',
     'volcano',
     'ballplayer, baseball player',
     'groom, bridegroom',
     'scuba diver',
     'rapeseed',
     'daisy',
     "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum",
     'corn',
     'acorn',
     'hip, rose hip, rosehip',
     'buckeye, horse chestnut, conker',
     'coral fungus',
     'agaric',
     'gyromitra',
     'stinkhorn, carrion fungus',
     'earthstar',
     'hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa',
     'bolete',
     'ear, spike, capitulum',
     'toilet tissue, toilet paper, bathroom tissue']

```python
len(imagenet_labels)
```

    1000

```python
img_path = Path("cat.jpg")
if not img_path.exists():
    img_path, _ = urlretrieve(
        'https://www.anishathalye.com/media/2017/07/25/cat.jpg',
        img_path
    )
img_class = 281
imagenet_labels[img_class]
```

    'tabby, tabby cat'

```python
img = Image.open(img_path)
#big_dim = max(img.width, img.height)
wide = img.width > img.height
new_w = 299 if not wide else int(img.width * 299 / img.height)
new_h = 299 if wide else int(img.height * 299 / img.width)
img = img.resize((new_w, new_h)).crop((0, 0, 299, 299))
#img = (np.asarray(img) / 255.0).astype(np.float32)
#img = np.asarray(img, dtype=np.float32)
img = np.asarray(img)
img.shape, img.dtype, img.max(), img.min()
```

    ((299, 299, 3), dtype('uint8'), 255, 0)

```python
def classify(img, correct_class=None, target_class=None, k=10):
    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 8))
    fig.sca(ax1)  # sca() Select Current Axis
    ax1.imshow(img)
    #p = inception_model(np.expand_dims(img, axis=0))[0]
    p = inception(img)
    p = p.numpy()
    #print(f"type(p) = {type(p)}, len(p) = {len(p)}")

    #fig.sca(ax1)  # Why should there be two fig.sca(ax1)?

    # display the top 10 prediceted classes
    #topk = list(p.argsort()[-10:][::-1])
    #topk = list(tf.argsort(p)[-10:][::-1])
    #k = 20
    topk = list((tf.argsort(p)[-k:][::-1]).numpy())
    #print(f"topk = {topk}")
    topprobs = p[topk]
    barlist = ax2.bar(range(k), topprobs)
    if target_class in topk:
        barlist[topk.index(target_class)].set_color('r')
    if correct_class in topk:
        barlist[topk.index(correct_class)].set_color('g')
    plt.sca(ax2)
    plt.ylim([0, 1.1])
    plt.xticks(range(k),
               [imagenet_labels[i][:15] for i in topk],
               rotation='vertical')
    # :15 means "taking the first 15 characters" for fear of long-string class.
    fig.subplots_adjust(bottom=0.2)
    plt.show()
```

```python
classify(img, correct_class=img_class)
```

```python
isinstance(tf.Variable(10), tf.Tensor)
```

    False

```python
isinstance(tf.Variable(10), tf.Variable)
```

    True

def inverse_preprocess(array, uint8=False):
    if isinstance(array, tf.Tensor) or isinstance(array, tf.Variable):
        array = array.numpy()
    array01 = (array + 1) / 2
    if uint8:
        return (array01 * 255).astype(np.uint8)
    else:
        return array01

```python
def inverse_preprocess(array, uint8=False):
    return (array + 1) / 2
```

# Adversarial examples

Given an image $\mathbf{x}$, our neural network outputs a probability distribution over labels, $P(y \mid \mathbf{x})$. When we craft an adversarial input, we want to find an $\hat{\mathbf{x}}$ where $\log P(\hat{y} \mid \hat{\mathbf{x}})$ is maximized for a target label $\hat{y}$: that way, our input will be misclassified as the target class. We can ensure that $\hat{\mathbf{x}}$ doesn't look too different from the original $\mathbf{x}$ by constraining ourselves to some $\ell_\infty$ <del>box</del> ball with radius $\epsilon$, requiring that $\left\lVert \mathbf{x} - \hat{\mathbf{x}} \right\rVert_\infty \le \epsilon$.

In this framework, an adversarial example is the solution to a constrained optimization problem that we can solve using [backpropagation](https://colah.github.io/posts/2015-08-Backprop/) and projected gradient descent, basically the same techniques that are used to train networks themselves. The algorithm is simple:

We begin by initializing our adversarial example as $\hat{\mathbf{x}} \leftarrow \mathbf{x}$. Then, we repeat the following until convergence:

1. $\hat{\mathbf{x}} \leftarrow \hat{\mathbf{x}} + \alpha \cdot \nabla \log P(\hat{y} \mid \hat{\mathbf{x}})$
2. $\hat{\mathbf{x}} \leftarrow \mathrm{clip}(\hat{\mathbf{x}}, \mathbf{x} - \epsilon, \mathbf{x} + \epsilon)$

```python
#image = tf.Variable(img, dtype=tf.float32)
image = tf.Variable(tf.zeros((299,299,3), dtype=tf.float32))
x_hat = image

x_hat.dtype, x_hat.shape
```

    (tf.float32, TensorShape([299, 299, 3]))

```python
demo_epsilon = 2.0/255.0 # a really small perturbation
# Test other target classes at your own will
#demo_target = imagenet_labels.index("pizza, pizza pie")
demo_target = imagenet_labels.index("desktop computer")
demo_epsilon, demo_target
```

    (0.00784313725490196, 527)

help(np.clip)

```python
img_processed = keras.applications.inception_v3.preprocess_input(img)
below = img_processed - demo_epsilon
below = np.clip(below, -1, 1)
above = img_processed + demo_epsilon
above = np.clip(above, -1, 1)
print(f"img[:2, :2, 0] =\n{img[:2, :2, 0]}\n")
print(f"img_processed[:2, :2, 0] =\n{img_processed[:2, :2, 0]}\n")
print(f"below[:2, :2, 0] =\n{below[:2, :2, 0]}\n")
print(f"above[:2, :2, 0] =\n{above[:2, :2, 0]}")
```

    img[:2, :2, 0] =
    [[14 14]
     [15 16]]

    img_processed[:2, :2, 0] =
    [[-0.8901961  -0.8901961 ]
     [-0.88235295 -0.8745098 ]]

    below[:2, :2, 0] =
    [[-0.8980392  -0.8980392 ]
     [-0.8901961  -0.88235295]]

    above[:2, :2, 0] =
    [[-0.88235295 -0.88235295]
     [-0.8745098  -0.8666667 ]]

demo_lr = 1e-1
demo_steps = 100

# projected gradient descent
for i in tqdm(range(demo_steps)):
    ## gradient descent step
    #_, loss_value = sess.run(
    #    [optim_step, loss],
    #    feed_dict={learning_rate: demo_lr, y_hat: demo_target})
    ## project step
    #sess.run(project_step, feed_dict={x: img, epsilon: demo_epsilon})
    with tf.GradientTape() as tape:
        #tape.watch(x_hat)
        proba = inception(x_hat)[demo_target]
        #loss = keras.losses.sparse_categorical_crossentropy()
        #log_likelihood = inception(image)[tf.constant(1)]

    grad = tape.gradient(proba, [x_hat])[0]
    #print(f"grad.shape = {grad.shape}")
    #print(f"demo_lr*grad = {type(demo_lr*grad)}")
    #tf.Variable.assign_add(x_hat, demo_lr*grad)
    #x_hat.assign_add(demo_lr*grad)
    #x_hat.assign_add(demo_lr*grad.numpy())
    #x_hat = x_hat + demo_lr*grad
    #projected = tf.clip_by_value(tf.clip_by_value(x_hat, below, above), 0, 1)
    #projected = tf.clip_by_value(
    #     tf.clip_by_value(x_hat + demo_lr * grad, below, above),
    #     -1, 1)
    #tf.Variable.assign(x_hat, projected)
    projected = tf.clip_by_value(x_hat + demo_lr * grad, below, above)
    x_hat.assign(projected)
    if (i+1) % 10 == 0:
        print(f'step{i+1: 4d}, proba={proba:.9f}')classify(
    inverse_preprocess(x_hat),
    correct_class=img_class,
    target_class=demo_target
)imagenet_labels[tf.argmax(inception(x_hat)).numpy()]demo_target, tf.argsort(inception(x_hat), direction="DESCENDING").numpy()[:10]

```python

```

```python

```

```python
demo_lr = 1e-1
demo_steps = 100

# projected gradient descent
for i in tqdm(range(demo_steps)):
    ## gradient descent step
    #_, loss_value = sess.run(
    #    [optim_step, loss],
    #    feed_dict={learning_rate: demo_lr, y_hat: demo_target})
    ## project step
    #sess.run(project_step, feed_dict={x: img, epsilon: demo_epsilon})
    with tf.GradientTape() as tape:
        #tape.watch(x_hat)
        #proba = inception(x_hat)[demo_target]
        probas = inception(x_hat)
        loss = keras.losses.sparse_categorical_crossentropy(demo_target, probas)
        #loss = keras.losses.sparse_categorical_crossentropy([demo_target], [proba])
        #log_likelihood = inception(image)[tf.constant(1)]

    grad = tape.gradient(loss, [x_hat])[0]
    #print(f"grad.shape = {grad.shape}")
    #print(f"demo_lr*grad = {type(demo_lr*grad)}")
    #tf.Variable.assign_add(x_hat, demo_lr*grad)

    #x_hat.assign_add(demo_lr*grad)
    #x_hat.assign_sub(demo_lr*grad)

    #x_hat.assign_add(demo_lr*grad.numpy())
    #x_hat = x_hat + demo_lr*grad
    projected = tf.clip_by_value(
        tf.clip_by_value(x_hat - demo_lr*grad, below, above),
        -1, 1)
    #projected = tf.clip_by_value(x_hat + demo_lr * grad, below, above)
    #tf.Variable.assign(x_hat, projected)
    x_hat.assign(projected)

    if (i+1) % 10 == 0:
        print(f'step{i+1: 4d}, loss={loss:.6f}, proba={probas[demo_target].numpy():.6f}')
```

      0%|          | 0/100 [00:00<?, ?it/s]

    step  10, loss=7.364683, proba=0.000633
    step  20, loss=5.993431, proba=0.002495
    step  30, loss=5.313797, proba=0.004923
    step  40, loss=4.772011, proba=0.008463
    step  50, loss=4.347112, proba=0.012944
    step  60, loss=3.992121, proba=0.018461
    step  70, loss=3.687208, proba=0.025042
    step  80, loss=3.429519, proba=0.032403
    step  90, loss=3.221476, proba=0.039896
    step 100, loss=2.995216, proba=0.050026

```python
classify(
    (inverse_preprocess(x_hat.numpy()) * 255).astype(np.uint8),
    #inverse_preprocess(x_hat.numpy()),
    correct_class=img_class,
    target_class=demo_target
)
```

```python
probas[demo_target]
```

    <tf.Tensor: shape=(), dtype=float32, numpy=0.05002581>

```python
imagenet_labels.index("Egyptian cat")
```

    285

```python
probas[imagenet_labels.index("Egyptian cat")]
```

    <tf.Tensor: shape=(), dtype=float32, numpy=0.025739914>

```python
demo_target, img_class
```

    (527, 281)

```python
tf.sort(probas, direction="DESCENDING")[:10]
```

    <tf.Tensor: shape=(10,), dtype=float32, numpy=
    array([0.12328403, 0.05002581, 0.02573991, 0.02450114, 0.02115966,
           0.01935675, 0.01694031, 0.0144507 , 0.01277901, 0.01170273],
          dtype=float32)>

```python
def top(array, k=10):
    return [imagenet_labels[index] for index in tf.argsort(probas, direction="DESCENDING")[:k]]
```

```python
top(probas)
```

    ['tabby, tabby cat',
     'desktop computer',
     'Egyptian cat',
     'tiger cat',
     'radio, wireless',
     'notebook, notebook computer',
     'laptop, laptop computer',
     'mouse, computer mouse',
     'monitor',
     'screen, CRT screen']

tf.argsort(probas, direction="DESCENDING")

```python

```

```python

```

```python

```

```python

```

```python

```

```python
import tensorflow.contrib.slim as slim
import tensorflow.contrib.slim.nets as nets
```

```python
tf.logging.set_verbosity(tf.logging.ERROR)
sess = tf.InteractiveSession()
```

First, we set up the input image. We use a `tf.Variable` instead of a `tf.placeholder` because we will need it to be trainable. We can still feed it when we want to.

```python
image = tf.Variable(tf.zeros((299, 299, 3)))
```

Next, we load the Inception v3 model.

```python
def inception(image, reuse):
    preprocessed = tf.multiply(tf.subtract(tf.expand_dims(image, 0), 0.5), 2.0)
    arg_scope = nets.inception.inception_v3_arg_scope(weight_decay=0.0)
    with slim.arg_scope(arg_scope):
        logits, _ = nets.inception.inception_v3(
            preprocessed, 1001, is_training=False, reuse=reuse)
        logits = logits[:,1:] # ignore background class
        probs = tf.nn.softmax(logits) # probabilities
    return logits, probs

logits, probs = inception(image, reuse=False)
```

The `preprocessed = tf.multiply(tf.subtract(tf.expand_dims(image, 0), 0.5), 2.0)` line, I guess, was just to make the output real numbers lying in $[-1, 1]$ instead of $[0, 1]$.

Next, we load pre-trained weights. This Inception v3 has a top-5 accuracy of 93.9%.

```python
import tempfile
from urllib.request import urlretrieve
#from urllib import urlretrieve
import tarfile
import os
```

In the original notebook, the url of the following cell were `https` but I don't know why it does not work any more; instead, one has to replace `https` by `http`.

```python
data_dir = tempfile.mkdtemp()
inception_tarball, _ = urlretrieve(
    'http://download.tensorflow.org/models/inception_v3_2016_08_28.tar.gz')
tarfile.open(inception_tarball, 'r:gz').extractall(data_dir)
```

**(?1)**
- `urlretrieve()`'s return value
- `tarfile.open`'s return value

```python
data_dir
```

```python
ls $data_dir
```
ls ${data_dir}ls: cannot access '$/tmp/tmpvzku6vpk': No such file or directoryls $(data_dir)/bin/bash: line 1: data_dir: command not found
02_remarked.ipynb  adversarial.ipynb  README.md  trash.py

```python
restore_vars = [
    var for var in tf.global_variables()
    if var.name.startswith('InceptionV3/')
]
saver = tf.train.Saver(restore_vars)
saver.restore(sess, os.path.join(data_dir, 'inception_v3.ckpt'))
```

Next, we write some code to show an image, classify it, and show the classification result.
!conda install -n homl1e matplotlib

```python
import json
import matplotlib.pyplot as plt
```

```python
imagenet_json, _ = urlretrieve(
    'https://www.anishathalye.com/media/2017/07/25/imagenet.json')
with open(imagenet_json) as f:
    imagenet_labels = json.load(f)
```

```python
imagenet_json
```

```python
imagenet_labels
```

```python
type(imagenet_labels), len(imagenet_labels)
```

```python
def classify(img, correct_class=None, target_class=None):
    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 8))
    fig.sca(ax1)  # sca() Select Current Axis
    p = sess.run(probs, feed_dict={image: img})[0]
    ax1.imshow(img)
    fig.sca(ax1)  # Why should there be two fig.sca(ax1)?

    # display the top 10 prediceted classes
    topk = list(p.argsort()[-10:][::-1])
    topprobs = p[topk]
    barlist = ax2.bar(range(10), topprobs)
    if target_class in topk:
        barlist[topk.index(target_class)].set_color('r')
    if correct_class in topk:
        barlist[topk.index(correct_class)].set_color('g')
    plt.sca(ax2)
    plt.ylim([0, 1.1])
    plt.xticks(range(10),
               [imagenet_labels[i][:15] for i in topk],
               rotation='vertical')
    # :15 means "taking the first 15 characters" for fear of long-string class.
    fig.subplots_adjust(bottom=0.2)
    plt.show()
```

```python
L = ["a", "b", "c"]
L.index("c"), L.index("b"), #L.index("z")
```

## Example image

We load our example image and make sure it's classified correctly.

```python
import PIL
import numpy as np
```

```python
img_path, _ = urlretrieve('https://www.anishathalye.com/media/2017/07/25/cat.jpg')
img_class = 281
img = PIL.Image.open(img_path)
big_dim = max(img.width, img.height)
wide = img.width > img.height
new_w = 299 if not wide else int(img.width * 299 / img.height)
new_h = 299 if wide else int(img.height * 299 / img.width)
img = img.resize((new_w, new_h)).crop((0, 0, 299, 299))
img = (np.asarray(img) / 255.0).astype(np.float32)
```

**(?)** What is the `new_w, new_h, resize()` all about?<br>
**(R)** The story was that Anish wanted to `crop((0,0,299,299))`, i.e. crop the `299x299` subimage for the upper left corner for each image. But the reality is that _**not every image has both width and height larger than**_ `299`. So the `resize((new_w, new_h))` was there to guarantee this. Indeed,
\begin{align}
  h_{\text{new}} = 299,\;  w_{\text{new}} = h_{\text{new}} \frac{w}{h} \quad\text{when}\quad h < w \\
  w_{\text{new}} = 299,\;  h_{\text{new}} = w_{\text{new}} \frac{h}{w} \quad\text{when}\quad h \ge w
\end{align}

which converted into words says **_always convert the shorter side to_** `299` and **_the longer side to its rightful length according to the original ratio_**.

```python
imagenet_labels[img_class]
```

```python
classify(img, correct_class=img_class)
```

# Adversarial examples

Given an image $\mathbf{x}$, our neural network outputs a probability distribution over labels, $P(y \mid \mathbf{x})$. When we craft an adversarial input, we want to find an $\hat{\mathbf{x}}$ where $\log P(\hat{y} \mid \hat{\mathbf{x}})$ is maximized for a target label $\hat{y}$: that way, our input will be misclassified as the target class. We can ensure that $\hat{\mathbf{x}}$ doesn't look too different from the original $\mathbf{x}$ by constraining ourselves to some $\ell_\infty$ <del>box</del> ball with radius $\epsilon$, requiring that $\left\lVert \mathbf{x} - \hat{\mathbf{x}} \right\rVert_\infty \le \epsilon$.

In this framework, an adversarial example is the solution to a constrained optimization problem that we can solve using [backpropagation](https://colah.github.io/posts/2015-08-Backprop/) and projected gradient descent, basically the same techniques that are used to train networks themselves. The algorithm is simple:

We begin by initializing our adversarial example as $\hat{\mathbf{x}} \leftarrow \mathbf{x}$. Then, we repeat the following until convergence:

1. $\hat{\mathbf{x}} \leftarrow \hat{\mathbf{x}} + \alpha \cdot \nabla \log P(\hat{y} \mid \hat{\mathbf{x}})$
2. $\hat{\mathbf{x}} \leftarrow \mathrm{clip}(\hat{\mathbf{x}}, \mathbf{x} - \epsilon, \mathbf{x} + \epsilon)$

## Initialization

We start with the easiest part: writing a TensorFlow op for initialization.

```python
x = tf.placeholder(tf.float32, (299, 299, 3))

x_hat = image # our trainable adversarial input
assign_op = tf.assign(x_hat, x)
```

## Gradient descent step

Next, we write the gradient descent step to maximize the log probability of the target class (or equivalently, minimize the [cross entropy](https://en.wikipedia.org/wiki/Cross_entropy)).

```python
type(())  # empty tuple as the shape of a float
```

```python
learning_rate = tf.placeholder(tf.float32, ())
y_hat = tf.placeholder(tf.int32, ())

labels = tf.one_hot(y_hat, 1000)
loss = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=[labels])
optim_step = tf.train.GradientDescentOptimizer(
    learning_rate).minimize(loss, var_list=[x_hat])
```

**N.B.** `logits` was defined in an earlier cell, via `image`, which should probably viewed as identical to `x_hat`.

**(?)** The `var_list` param of `minimize()`

## Projection step

Finally, we write the projection step to keep our adversarial example visually close to the original image. Additionally, we clip to $[0, 1]$ to keep it a valid image.

```python
epsilon = tf.placeholder(tf.float32, ())

below = x - epsilon
above = x + epsilon
projected = tf.clip_by_value(tf.clip_by_value(x_hat, below, above), 0, 1)
with tf.control_dependencies([projected]):
    # cf. p.323 ageron's homl1e
    project_step = tf.assign(x_hat, projected)
```

**(?)** What would happen if we do not use `tf.control_dependencies()` here?

## Execution

Finally, we're ready to synthesize an adversarial example. We arbitrarily choose "guacamole" (imagenet class 924) as our target class.

```python
imagenet_labels.index("pizza, pizza pie")
```

```python
demo_epsilon = 2.0/255.0 # a really small perturbation
demo_lr = 1e-1
demo_steps = 100
demo_target = 924 # "guacamole"
# Test other target classes at your own will
demo_target = imagenet_labels.index("pizza, pizza pie")

# initialization step
sess.run(assign_op, feed_dict={x: img})

# projected gradient descent
for i in range(demo_steps):
    # gradient descent step
    _, loss_value = sess.run(
        [optim_step, loss],
        feed_dict={learning_rate: demo_lr, y_hat: demo_target})
    # project step
    sess.run(project_step, feed_dict={x: img, epsilon: demo_epsilon})
    if (i+1) % 10 == 0:
        print('step %d, loss=%g' % (i+1, loss_value))

adv = x_hat.eval() # retrieve the adversarial example
```

**Rmk.**
- Took several minutes to finish running `100` steps on Thinkpad X200.
- [%g](https://stackoverflow.com/questions/30658919/the-precision-of-printf-with-specifier-g)
- Note that `image` and `x_hat` really are regarded as identical in the TensorFlow graph above, because
  - `project_step` assigns the projection to `x_hat`
  - Optimization updates `image`
  - They have to be the same thing in order for this to work.

This adversarial image is visually indistinguishable from the original, with no visual artifacts. However, it's classified as "guacamole" with high probability!

```python
classify(adv, correct_class=img_class, target_class=demo_target)
```

**(?)** How small was the perturbation `2.0/255.0`? Try to do an experiment with random noise of that magnitude of perturbation.

**Stopped here (2021/02/19 (金) 16h04)**

This adv attack is very cool, but we haven't verified that the weights we downloaded really classify the imagenet images well. Let's do this.
import tensorflow_datasets as tfds
ds = tfds.load("imagenet2012_subset/1pct")
ds---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
<ipython-input-32-188e57836169> in <module>
----> 1 import tensorflow_datasets as tfds
      2 ds = tfds.load("imagenet2012_subset/1pct")
      3 ds

ModuleNotFoundError: No module named 'tensorflow_datasets'!conda install -n homl1e tensorflow-datasets -y!which pip!pip install tensorflow_datasets!pip uninstall -y tensorflow_datasetsimport tensorflow_datasets as tfds
ds, ds_info = tfds.load("imagenet2012_subset/1pct", with_info=True)
ds_info

```python
tf.__version__
```

It seems that one cannot use `tfds` in `tensorflow==1.13.1`.<br>
`pip` seems to download ok, but cannot run.

# Robust adversarial examples

Now, we go through a more advanced example. We follow our approach for [synthesizing robust adversarial examples](https://arxiv.org/abs/1707.07397) to find a single perturbation of our cat image that's simultaneously adversarial under some chosen distribution of transformations.  We could choose any distribution of differentiable transformations; in this post, we'll synthesize a single adversarial input that's robust to rotation by $\theta \in [-\pi/4, \pi/4]$.

Before we proceed, let's check if our previous example is still adversarial if we rotate it, say by an angle of $\theta = \pi/8$.

```python
ex_angle = np.pi/8

angle = tf.placeholder(tf.float32, ())
rotated_image = tf.contrib.image.rotate(image, angle)
rotated_example = rotated_image.eval(feed_dict={image: adv, angle: ex_angle})
classify(rotated_example, correct_class=img_class, target_class=demo_target)
```

Looks like our original adversarial example is not rotation-invariant! Let's try one more.

```python
rotated_example = rotated_image.eval(feed_dict={image: adv, angle: -np.pi/4})
classify(rotated_example, correct_class=img_class, target_class=demo_target)
```

So, how do we make an adversarial example robust to a distribution of transformations? Given some distribution of transformations $T$, we can maximize $\mathbb{E}_{t \sim T} \log P\left(\hat{y} \mid t(\hat{\mathbf{x}})\right)$, subject to $\left\lVert \mathbf{x} - \hat{\mathbf{x}} \right\rVert_\infty \le \epsilon$. We can solve this optimization problem via projected gradient descent, noting that $\nabla \mathbb{E}_{t \sim T} \log P\left(\hat{y} \mid t(\hat{\mathbf{x}})\right)$ is $\mathbb{E}_{t \sim T} \nabla \log P\left(\hat{y} \mid t(\hat{\mathbf{x}})\right)$ and approximating with samples at each gradient descent step.

Rather than manually implementing the gradient sampling, we can use a trick to get TensorFlow to do it for us: we can model our sampling-based gradient descent as doing gradient descent over an ensemble of stochastic classifiers that randomly sample from the distribution and transform their input before classifying it.

**(?)** Why does Anish keep speak of **distribution of transformations**?

```python
num_samples = 10
average_loss = 0
for i in range(num_samples):
    rotated = tf.contrib.image.rotate(
        image, tf.random_uniform((), minval=-np.pi/4, maxval=np.pi/4))
    rotated_logits, _ = inception(rotated, reuse=True)
    average_loss += tf.nn.softmax_cross_entropy_with_logits(
        logits=rotated_logits, labels=labels) / num_samples
```

**(?)** What is `reuse=True` here? It means that we reuse the weights we've found before?

**(?)** Why `average_loss` has to be divided by `num_samples` here?

**(?)** Why in the last `tf.nn.softmax_cross_entropy_with_logits()` we had `labels=[labels]` while in this one we have `labels=labels`?

We can reuse our `assign_op` and `project_step`, though we'll have to write a new `optim_step` for this new objective.

```python
optim_step = tf.train.GradientDescentOptimizer(
    learning_rate).minimize(average_loss, var_list=[x_hat])
```

Finally, we're ready to run PGD to generate our adversarial input. As in the previous example, we'll choose "guacamole" as our target class.

```python
demo_epsilon = 8.0/255.0 # still a pretty small perturbation
demo_lr = 2e-1
demo_steps = 300
demo_target = 924 # "guacamole"

# initialization step
sess.run(assign_op, feed_dict={x: img})

# projected gradient descent
for i in range(demo_steps):
    # gradient descent step
    _, loss_value = sess.run(
        [optim_step, average_loss],
        feed_dict={learning_rate: demo_lr, y_hat: demo_target})
    # project step
    sess.run(project_step, feed_dict={x: img, epsilon: demo_epsilon})
    if (i+1) % 50 == 0:
        print('step %d, loss=%g' % (i+1, loss_value))

adv_robust = x_hat.eval() # retrieve the adversarial example
```

This adversarial image is classified as "guacamole" with high confidence, even when it's rotated!

```python
rotated_example = rotated_image.eval(feed_dict={image: adv_robust, angle: ex_angle})
#rotated_example = rotated_image.eval(feed_dict={image: adv_robust, angle: np.random.uniform(-np.pi/4, np.pi/4)})
classify(rotated_example, correct_class=img_class, target_class=demo_target)
```

## Evaluation

Let's examine the rotation-invariance of the robust adversarial example we produced over the entire range of angles, looking at $P(\hat{y} \mid \hat{\mathbf{x}})$ over $\theta \in [-\pi/4, \pi/4]$.

```python
thetas = np.linspace(-np.pi/4, np.pi/4, 301)

p_naive = []
p_robust = []
for theta in thetas:
    rotated = rotated_image.eval(feed_dict={image: adv_robust, angle: theta})
    p_robust.append(probs.eval(feed_dict={image: rotated})[0][demo_target])

    rotated = rotated_image.eval(feed_dict={image: adv, angle: theta})
    p_naive.append(probs.eval(feed_dict={image: rotated})[0][demo_target])

robust_line, = plt.plot(thetas, p_robust, color='b', linewidth=2, label='robust')
naive_line, = plt.plot(thetas, p_naive, color='r', linewidth=2, label='naive')
plt.ylim([0, 1.05])
plt.xlabel('rotation angle')
plt.ylabel('target class probability')
plt.legend(handles=[robust_line, naive_line], loc='lower right')
plt.show()
```

It's super effective!

## Challenge
It would be a good challenge to write this same notebook using
- `tf2`
- `torch`, etc.

```python

```


# VQE for Unitary Coupled Cluster using tket

In this tutorial, we will focus on:

- building parameterised ansätze for variational algorithms;

- compilation tools for UCC-style ansätze.

This example assumes the reader is familiar with the Variational Quantum Eigensolver and its application to electronic structure problems through the Unitary Coupled Cluster approach.

To run this example, you will need `pytket` and `pytket-qiskit`, as well as `openfermion`, `scipy`, and `sympy`.

We will start with a basic implementation and then gradually modify it to make it faster, more general, and less noisy. The final solution is given in full at the bottom of the notebook.

Suppose we have some electronic configuration problem, expressed via a physical Hamiltonian. (The Hamiltonian and excitations in this example were obtained using `qiskit-aqua` version 0.5.2 and `pyscf` for H2, bond length 0.75A, sto3g basis, Jordan-Wigner encoding, with no qubit reduction or orbital freezing.)

```python
from openfermion import QubitOperator
```

```python
hamiltonian = (
    -0.8153001706270075 * QubitOperator("")
    + 0.16988452027940318 * QubitOperator("Z0")
    + -0.21886306781219608 * QubitOperator("Z1")
    + 0.16988452027940323 * QubitOperator("Z2")
    + -0.2188630678121961 * QubitOperator("Z3")
    + 0.12005143072546047 * QubitOperator("Z0 Z1")
    + 0.16821198673715723 * QubitOperator("Z0 Z2")
    + 0.16549431486978672 * QubitOperator("Z0 Z3")
    + 0.16549431486978672 * QubitOperator("Z1 Z2")
    + 0.1739537877649417 * QubitOperator("Z1 Z3")
    + 0.12005143072546047 * QubitOperator("Z2 Z3")
    + 0.04544288414432624 * QubitOperator("X0 X1 X2 X3")
    + 0.04544288414432624 * QubitOperator("X0 X1 Y2 Y3")
    + 0.04544288414432624 * QubitOperator("Y0 Y1 X2 X3")
    + 0.04544288414432624 * QubitOperator("Y0 Y1 Y2 Y3")
)
nuclear_repulsion_energy = 0.70556961456
```

We would like to define our ansatz for arbitrary parameter values. For simplicity, let's start with a Hardware Efficient Ansatz.

```python
from pytket import Circuit
```

Hardware efficient ansatz:

```python
def hea(params):
    ansatz = Circuit(4)
    for i in range(4):
        ansatz.Ry(params[i], i)
    for i in range(3):
        ansatz.CX(i, i + 1)
    for i in range(4):
        ansatz.Ry(params[4 + i], i)
    return ansatz
```

We can use this to build the objective function for our optimisation.

```python
from pytket.extensions.qiskit import AerBackend
from pytket.utils import expectation_from_counts
```

```python
backend = AerBackend()
```

Naive objective function:

```python
def objective(params):
    energy = 0
    for term, coeff in hamiltonian.terms.items():
        if not term:
            energy += coeff
            continue
        circ = hea(params)
        circ.add_c_register("c", len(term))
        for i, (q, pauli) in enumerate(term):
            if pauli == "X":
                circ.H(q)
            elif pauli == "Y":
                circ.V(q)
            circ.Measure(q, i)
        backend.compile_circuit(circ)
        counts = backend.run_circuit(circ, n_shots=4000).get_counts()
        energy += coeff * expectation_from_counts(counts)
    return energy + nuclear_repulsion_energy
```

This objective function is then run through a classical optimiser to find the set of parameter values that minimise the energy of the system. For the sake of example, we will just run this with a single parameter value.

```python
arg_values = [
    -7.31158201e-02,
    -1.64514836e-04,
    1.12585591e-03,
    -2.58367544e-03,
    1.00006068e00,
    -1.19551357e-03,
    9.99963988e-01,
    2.53283285e-03,
]
```

```python
energy = objective(arg_values)
print(energy)
```

The HEA is designed to cram as many orthogonal degrees of freedom into a small circuit as possible to be able to explore a large region of the Hilbert space whilst the circuits themselves can be run with minimal noise. These ansätze give virtually-optimal circuits by design, but suffer from an excessive number of variational parameters making convergence slow, barren plateaus where the classical optimiser fails to make progress, and spanning a space where most states lack a physical interpretation. These drawbacks can necessitate adding penalties and may mean that the ansatz cannot actually express the true ground state.

The UCC ansatz, on the other hand, is derived from the electronic configuration. It sacrifices efficiency of the circuit for the guarantee of physical states and the variational parameters all having some meaningful effect, which helps the classical optimisation to converge.

This starts by defining the terms of our single and double excitations. These would usually be generated using the orbital configurations, so we will just use a hard-coded example here for the purposes of demonstration.

```python
from pytket.pauli import Pauli, QubitPauliString
from pytket.circuit import Qubit
```

```python
q = [Qubit(i) for i in range(4)]
xyii = QubitPauliString([q[0], q[1]], [Pauli.X, Pauli.Y])
yxii = QubitPauliString([q[0], q[1]], [Pauli.Y, Pauli.X])
iixy = QubitPauliString([q[2], q[3]], [Pauli.X, Pauli.Y])
iiyx = QubitPauliString([q[2], q[3]], [Pauli.Y, Pauli.X])
xxxy = QubitPauliString(q, [Pauli.X, Pauli.X, Pauli.X, Pauli.Y])
xxyx = QubitPauliString(q, [Pauli.X, Pauli.X, Pauli.Y, Pauli.X])
xyxx = QubitPauliString(q, [Pauli.X, Pauli.Y, Pauli.X, Pauli.X])
yxxx = QubitPauliString(q, [Pauli.Y, Pauli.X, Pauli.X, Pauli.X])
yyyx = QubitPauliString(q, [Pauli.Y, Pauli.Y, Pauli.Y, Pauli.X])
yyxy = QubitPauliString(q, [Pauli.Y, Pauli.Y, Pauli.X, Pauli.Y])
yxyy = QubitPauliString(q, [Pauli.Y, Pauli.X, Pauli.Y, Pauli.Y])
xyyy = QubitPauliString(q, [Pauli.X, Pauli.Y, Pauli.Y, Pauli.Y])
```

```python
singles_a = {xyii: 1.0, yxii: -1.0}
singles_b = {iixy: 1.0, iiyx: -1.0}
doubles = {
    xxxy: 0.25,
    xxyx: -0.25,
    xyxx: 0.25,
    yxxx: -0.25,
    yyyx: -0.25,
    yyxy: 0.25,
    yxyy: -0.25,
    xyyy: 0.25,
}
```

Building the ansatz circuit itself is often done naively by defining the map from each term down to basic gates and then applying it to each term.

```python
def add_operator_term(circuit: Circuit, term: QubitPauliString, angle: float):
    qubits = []
    for q, p in term.map.items():
        if p != Pauli.I:
            qubits.append(q)
            if p == Pauli.X:
                circuit.H(q)
            elif p == Pauli.Y:
                circuit.V(q)
    for i in range(len(qubits) - 1):
        circuit.CX(i, i + 1)
    circuit.Rz(angle, len(qubits) - 1)
    for i in reversed(range(len(qubits) - 1)):
        circuit.CX(i, i + 1)
    for q, p in term.map.items():
        if p == Pauli.X:
            circuit.H(q)
        elif p == Pauli.Y:
            circuit.Vdg(q)
```

Unitary Coupled Cluster Singles & Doubles ansatz:

```python
def ucc(params):
    ansatz = Circuit(4)
    # Set initial reference state
    ansatz.X(1).X(3)
    # Evolve by excitations
    for term, coeff in singles_a.items():
        add_operator_term(ansatz, term, coeff * params[0])
    for term, coeff in singles_b.items():
        add_operator_term(ansatz, term, coeff * params[1])
    for term, coeff in doubles.items():
        add_operator_term(ansatz, term, coeff * params[2])
    return ansatz
```

This is already quite verbose, but `pytket` has a neat shorthand construction for these operator terms using the `PauliExpBox` construction. We can then decompose these into basic gates using the `DecomposeBoxes` compiler pass.

```python
from pytket.circuit import PauliExpBox
from pytket.passes import DecomposeBoxes
```

```python
def add_excitation(circ, term_dict, param):
    for term, coeff in term_dict.items():
        qubits, paulis = zip(*term.map.items())
        pbox = PauliExpBox(paulis, coeff * param)
        circ.add_pauliexpbox(pbox, qubits)
```

UCC ansatz with syntactic shortcuts:

```python
def ucc(params):
    ansatz = Circuit(4)
    ansatz.X(1).X(3)
    add_excitation(ansatz, singles_a, params[0])
    add_excitation(ansatz, singles_b, params[1])
    add_excitation(ansatz, doubles, params[2])
    DecomposeBoxes().apply(ansatz)
    return ansatz
```

The objective function can also be simplified using a utility method for constructing the measurement circuits and processing for expectation value calculations.

```python
from pytket.utils.operators import QubitPauliOperator
from pytket.utils import get_operator_expectation_value
```

```python
hamiltonian_op = QubitPauliOperator.from_OpenFermion(hamiltonian)
```

Simplified objective function using utilities:

```python
def objective(params):
    circ = ucc(params)
    return (
        get_operator_expectation_value(circ, hamiltonian_op, backend, n_shots=4000)
        + nuclear_repulsion_energy
    )
```

```python
arg_values = [-3.79002933e-05, 2.42964799e-05, 4.63447157e-01]
```

```python
energy = objective(arg_values)
print(energy)
```

This is now the simplest form that this operation can take, but it isn't necessarily the most effective. When we decompose the ansatz circuit into basic gates, it is still very expensive. We can employ some of the circuit simplification passes available in `pytket` to reduce its size and improve fidelity in practice.

A good example is to decompose each `PauliExpBox` into basic gates and then apply `FullPeepholeOptimise`, which defines a compilation strategy utilising all of the simplifications in `pytket` that act locally on small regions of a circuit. We can examine the effectiveness by looking at the number of two-qubit gates before and after simplification, which tends to be a good indicator of fidelity for near-term systems where these gates are often slow and inaccurate.

```python
from pytket import OpType
from pytket.passes import FullPeepholeOptimise
```

```python
test_circuit = ucc(arg_values)
```

```python
print("CX count before", test_circuit.n_gates_of_type(OpType.CX))
print("CX depth before", test_circuit.depth_by_type(OpType.CX))
```

```python
FullPeepholeOptimise().apply(test_circuit)
```

```python
print("CX count after FPO", test_circuit.n_gates_of_type(OpType.CX))
print("CX depth after FPO", test_circuit.depth_by_type(OpType.CX))
```

These simplification techniques are very general and are almost always beneficial to apply to a circuit if you want to eliminate local redundancies. But UCC ansätze have extra structure that we can exploit further. They are defined entirely out of exponentiated tensors of Pauli matrices, giving the regular structure described by the `PauliExpBox`es. Under many circumstances, it is more efficient to not synthesise these constructions individually, but simultaneously in groups. The `PauliSimp` pass finds the description of a given circuit as a sequence of `PauliExpBox`es and resynthesises them (by default, in groups of commuting terms). This can cause great change in the overall structure and shape of the circuit, enabling the identification and elimination of non-local redundancy.

```python
from pytket.passes import PauliSimp
```

```python
test_circuit = ucc(arg_values)
```

```python
print("CX count before", test_circuit.n_gates_of_type(OpType.CX))
print("CX depth before", test_circuit.depth_by_type(OpType.CX))
```

```python
PauliSimp().apply(test_circuit)
```

```python
print("CX count after PS", test_circuit.n_gates_of_type(OpType.CX))
print("CX depth after PS", test_circuit.depth_by_type(OpType.CX))
```

```python
FullPeepholeOptimise().apply(test_circuit)
```

```python
print("CX count after PS+FPO", test_circuit.n_gates_of_type(OpType.CX))
print("CX depth after PS+FPO", test_circuit.depth_by_type(OpType.CX))
```

To include this into our routines, we can just add the simplification passes to the objective function. The `get_operator_expectation_value` utility handles compiling to meet the requirements of the backend, so we don't have to worry about that here.

Objective function with circuit simplification:

```python
def objective(params):
    circ = ucc(params)
    PauliSimp().apply(circ)
    FullPeepholeOptimise().apply(circ)
    return (
        get_operator_expectation_value(circ, hamiltonian_op, backend, n_shots=4000)
        + nuclear_repulsion_energy
    )
```

These circuit simplification techniques have tried to preserve the exact unitary of the circuit, but there are ways to change the unitary whilst preserving the correctness of the algorithm as a whole.

For example, the excitation terms are generated by trotterisation of the excitation operator, and the order of the terms does not change the unitary in the limit of many trotter steps, so in this sense we are free to sequence the terms how we like and it is sensible to do this in a way that enables efficient synthesis of the circuit. Prioritising collecting terms into commuting sets is a very beneficial heuristic for this and can be performed using the `gen_term_sequence_circuit` method to group the terms together into collections of `PauliExpBox`es and the `GuidedPauliSimp` pass to utilise these sets for synthesis.

```python
from pytket.passes import GuidedPauliSimp
from pytket.utils import gen_term_sequence_circuit
```

```python
def ucc(params):
    singles_params = {qps: params[0] * coeff for qps, coeff in singles.items()}
    doubles_params = {qps: params[1] * coeff for qps, coeff in doubles.items()}
    excitation_op = QubitPauliOperator({**singles_params, **doubles_params})
    reference_circ = Circuit(4).X(1).X(3)
    ansatz = gen_term_sequence_circuit(excitation_op, reference_circ)
    GuidedPauliSimp().apply(ansatz)
    FullPeepholeOptimise().apply(ansatz)
    return ansatz
```

Adding these simplification routines doesn't come for free. Compiling and simplifying the circuit to achieve the best results possible can be a difficult task, which can take some time for the classical computer to perform.

During a VQE run, we will call this objective function many times and run many measurement circuits within each, but the circuits that are run on the quantum computer are almost identical, having the same gate structure but with different gate parameters and measurements. We have already exploited this within the body of the objective function by simplifying the ansatz circuit before we call `get_operator_expectation_value`, so it is only done once per objective calculation rather than once per measurement circuit.

We can go even further by simplifying it once outside of the objective function, and then instantiating the simplified ansatz with the parameter values needed. For this, we will construct the UCC ansatz circuit using symbolic (parametric) gates.

```python
from sympy import symbols
```

Symbolic UCC ansatz generation:

```python
syms = symbols("p0 p1 p2")
singles_a_syms = {qps: syms[0] * coeff for qps, coeff in singles_a.items()}
singles_b_syms = {qps: syms[1] * coeff for qps, coeff in singles_b.items()}
doubles_syms = {qps: syms[2] * coeff for qps, coeff in doubles.items()}
excitation_op = QubitPauliOperator({**singles_a_syms, **singles_b_syms, **doubles_syms})
ucc_ref = Circuit(4).X(1).X(3)
ucc = gen_term_sequence_circuit(excitation_op, ucc_ref)
GuidedPauliSimp().apply(ucc)
FullPeepholeOptimise().apply(ucc)
```

Objective function using the symbolic ansatz:

```python
def objective(params):
    circ = ucc.copy()
    sym_map = dict(zip(syms, params))
    circ.symbol_substitution(sym_map)
    return (
        get_operator_expectation_value(circ, hamiltonian_op, backend, n_shots=4000)
        + nuclear_repulsion_energy
    )
```

We have now got some very good use of `pytket` for simplifying each individual circuit used in our experiment and for minimising the amount of time spent compiling, but there is still more we can do in terms of reducing the amount of work the quantum computer has to do. Currently, each (non-trivial) term in our measurement hamiltonian is measured by a different circuit within each expectation value calculation. Measurement reduction techniques exist for identifying when these observables commute and hence can be simultaneously measured, reducing the number of circuits required for the full expectation value calculation.

This is built in to the `get_operator_expectation_value` method and can be applied by specifying a way to partition the measuremrnt terms. `PauliPartitionStrat.CommutingSets` can greatly reduce the number of measurement circuits by combining any number of terms that mutually commute. However, this involves potentially adding an arbitrary Clifford circuit to change the basis of the measurements which can be costly on NISQ devices, so `PauliPartitionStrat.NonConflictingSets` trades off some of the reduction in circuit number to guarantee that only single-qubit gates are introduced.

```python
from pytket.partition import PauliPartitionStrat
```

Objective function using measurement reduction:

```python
def objective(params):
    circ = ucc.copy()
    sym_map = dict(zip(syms, params))
    circ.symbol_substitution(sym_map)
    return (
        get_operator_expectation_value(
            circ,
            operator,
            backend,
            n_shots=4000,
            partition_strat=PauliPartitionStrat.CommutingSets,
        )
        + nuclear_repulsion_energy
    )
```

At this point, we have completely transformed how our VQE objective function works, improving its resilience to noise, cutting the number of circuits run, and maintaining fast runtimes. In doing this, we have explored a number of the features `pytket` offers that are beneficial to VQE and the UCC method:

- high-level syntactic constructs for evolution operators;

- utility methods for easy expectation value calculations;

- both generic and domain-specific circuit simplification methods;

- symbolic circuit compilation;

- measurement reduction for expectation value calculations.

For the sake of completeness, the following gives the full code for the final solution, including passing the objective function to a classical optimiser to find the ground state:

```python
from openfermion import QubitOperator
from scipy.optimize import minimize
from sympy import symbols
```

```python
from pytket.extensions.qiskit import AerBackend
from pytket.circuit import Circuit, Qubit
from pytket.partition import PauliPartitionStrat
from pytket.passes import GuidedPauliSimp, FullPeepholeOptimise
from pytket.pauli import Pauli, QubitPauliString
from pytket.utils import get_operator_expectation_value, gen_term_sequence_circuit
from pytket.utils.operators import QubitPauliOperator
```

Obtain electronic Hamiltonian:

```python
hamiltonian = (
    -0.8153001706270075 * QubitOperator("")
    + 0.16988452027940318 * QubitOperator("Z0")
    + -0.21886306781219608 * QubitOperator("Z1")
    + 0.16988452027940323 * QubitOperator("Z2")
    + -0.2188630678121961 * QubitOperator("Z3")
    + 0.12005143072546047 * QubitOperator("Z0 Z1")
    + 0.16821198673715723 * QubitOperator("Z0 Z2")
    + 0.16549431486978672 * QubitOperator("Z0 Z3")
    + 0.16549431486978672 * QubitOperator("Z1 Z2")
    + 0.1739537877649417 * QubitOperator("Z1 Z3")
    + 0.12005143072546047 * QubitOperator("Z2 Z3")
    + 0.04544288414432624 * QubitOperator("X0 X1 X2 X3")
    + 0.04544288414432624 * QubitOperator("X0 X1 Y2 Y3")
    + 0.04544288414432624 * QubitOperator("Y0 Y1 X2 X3")
    + 0.04544288414432624 * QubitOperator("Y0 Y1 Y2 Y3")
)
nuclear_repulsion_energy = 0.70556961456
```

```python
hamiltonian_op = QubitPauliOperator.from_OpenFermion(hamiltonian)
```

Obtain terms for single and double excitations:

```python
q = [Qubit(i) for i in range(4)]
xyii = QubitPauliString([q[0], q[1]], [Pauli.X, Pauli.Y])
yxii = QubitPauliString([q[0], q[1]], [Pauli.Y, Pauli.X])
iixy = QubitPauliString([q[2], q[3]], [Pauli.X, Pauli.Y])
iiyx = QubitPauliString([q[2], q[3]], [Pauli.Y, Pauli.X])
xxxy = QubitPauliString(q, [Pauli.X, Pauli.X, Pauli.X, Pauli.Y])
xxyx = QubitPauliString(q, [Pauli.X, Pauli.X, Pauli.Y, Pauli.X])
xyxx = QubitPauliString(q, [Pauli.X, Pauli.Y, Pauli.X, Pauli.X])
yxxx = QubitPauliString(q, [Pauli.Y, Pauli.X, Pauli.X, Pauli.X])
yyyx = QubitPauliString(q, [Pauli.Y, Pauli.Y, Pauli.Y, Pauli.X])
yyxy = QubitPauliString(q, [Pauli.Y, Pauli.Y, Pauli.X, Pauli.Y])
yxyy = QubitPauliString(q, [Pauli.Y, Pauli.X, Pauli.Y, Pauli.Y])
xyyy = QubitPauliString(q, [Pauli.X, Pauli.Y, Pauli.Y, Pauli.Y])
```

Symbolic UCC ansatz generation:

```python
syms = symbols("p0 p1 p2")
singles_syms = {xyii: syms[0], yxii: -syms[0], iixy: syms[1], iiyx: -syms[1]}
doubles_syms = {
    xxxy: 0.25 * syms[2],
    xxyx: -0.25 * syms[2],
    xyxx: 0.25 * syms[2],
    yxxx: -0.25 * syms[2],
    yyyx: -0.25 * syms[2],
    yyxy: 0.25 * syms[2],
    yxyy: -0.25 * syms[2],
    xyyy: 0.25 * syms[2],
}
excitation_op = QubitPauliOperator({**singles_syms, **doubles_syms})
ucc_ref = Circuit(4).X(0).X(2)
ucc = gen_term_sequence_circuit(excitation_op, ucc_ref)
```

Circuit simplification:

```python
GuidedPauliSimp().apply(ucc)
FullPeepholeOptimise().apply(ucc)
```

Connect to a simulator/device:

```python
backend = AerBackend()
```

Objective function:

```python
def objective(params):
    circ = ucc.copy()
    sym_map = dict(zip(syms, params))
    circ.symbol_substitution(sym_map)
    return (
        get_operator_expectation_value(
            circ,
            hamiltonian_op,
            backend,
            n_shots=4000,
            partition_strat=PauliPartitionStrat.CommutingSets,
        )
        + nuclear_repulsion_energy
    ).real
```

Optimise against the objective function:

```python
initial_params = [1e-4, 1e-4, 4e-1]
result = minimize(objective, initial_params, method="Nelder-Mead")
print("Final parameter values", result.x)
print("Final energy value", result.fun)
```

Exercises:

- Replace the `get_operator_expectation_value` call with its implementation and use this to pull the analysis for measurement reduction outside of the objective function, so our circuits can be fully determined and compiled once. This means that the `symbol_substitution` method will need to be applied to each measurement circuit instead of just the state preparation circuit.

- Use the `SpamCorrecter` class to add some mitigation of the measurement errors. Start by running the characterisation circuits first, before your main VQE loop, then apply the mitigation to each of the circuits run within the objective function.

- Change the `backend` by passing in a `Qiskit` `NoiseModel` to simulate a noisy device. Compare the accuracy of the objective function both with and without the circuit simplification. Try running a classical optimiser over the objective function and compare the convergence rates with different noise models. If you have access to a QPU, try changing the `backend` to connect to that and compare the results to the simulator.