diff --git a/docs/_static/css/demo.css b/docs/_static/css/demo.css new file mode 100644 index 0000000000..86c718ea93 --- /dev/null +++ b/docs/_static/css/demo.css @@ -0,0 +1,91 @@ +.clr { + clear: both; +} + +.tab { + overflow: hidden; + border: 1px solid #ccc; + background-color: #f1f1f1; +} + +.tab button { + background-color: inherit; + font-size: 20px; + float: left; + border: none; + outline: none; + cursor: pointer; + padding: 14px 16px; + transition: 0.3s; +} + +.tab button:hover { + background-color: #ddd; +} + +.tab button.selected { + background-color: #ccc; +} + +.video_container { + float: left; + width: 50%; + height: 0px; + padding-top: 33.4%; /* 50% * (480/720) */ + position: relative; +} + +.display { + width: 100%; + height: 100%; + position:absolute; + top: 0; + left: 0; + border: 2px solid black; + box-sizing: border-box; +} + +.canvas_container { + float:right; + width: 50%; + height: 0px; + padding-top: 33.4%; + position: relative; +} + +#camera_prompt { + float: left; + margin: 5px; +} + +#camera_select { + float: left; + margin: 13px; +} + +#predict_button { + background-color: white; + color: black; + font-size: 25px; + padding: 10px 15px; + border: none; + border-radius: 10px; +} + +#predict_button:hover { + background-color: red; + color: white; +} + +#classification_left, #classification_right { + display: inline-block; +} + +.classification_label { + margin: 10px; +} + +.classification_prob { + display: inline-block; + margin: 5px; +} diff --git a/docs/_static/js/labels.js b/docs/_static/js/labels.js new file mode 100644 index 0000000000..2a5643d585 --- /dev/null +++ b/docs/_static/js/labels.js @@ -0,0 +1,1107 @@ +const image_net_labels = [ + "tench", + "goldfish", + "great white shark", + "tiger shark", + "hammerhead shark", + "electric ray", + "stingray", + "cock", + "hen", + "ostrich", + "brambling", + "goldfinch", + "house finch", + "junco", + "indigo bunting", + "American robin", + "bulbul", + "jay", + "magpie", + "chickadee", + "American dipper", + "kite", + "bald eagle", + "vulture", + "great grey owl", + "fire salamander", + "smooth newt", + "newt", + "spotted salamander", + "axolotl", + "American bullfrog", + "tree frog", + "tailed frog", + "loggerhead sea turtle", + "leatherback sea turtle", + "mud turtle", + "terrapin", + "box turtle", + "banded gecko", + "green iguana", + "Carolina anole", + "desert grassland whiptail lizard", + "agama", + "frilled-necked lizard", + "alligator lizard", + "Gila monster", + "European green lizard", + "chameleon", + "Komodo dragon", + "Nile crocodile", + "American alligator", + "triceratops", + "worm snake", + "ring-necked snake", + "eastern hog-nosed snake", + "smooth green snake", + "kingsnake", + "garter snake", + "water snake", + "vine snake", + "night snake", + "boa constrictor", + "African rock python", + "Indian cobra", + "green mamba", + "sea snake", + "Saharan horned viper", + "eastern diamondback rattlesnake", + "sidewinder", + "trilobite", + "harvestman", + "scorpion", + "yellow garden spider", + "barn spider", + "European garden spider", + "southern black widow", + "tarantula", + "wolf spider", + "tick", + "centipede", + "black grouse", + "ptarmigan", + "ruffed grouse", + "prairie grouse", + "peacock", + "quail", + "partridge", + "grey parrot", + "macaw", + "sulphur-crested cockatoo", + "lorikeet", + "coucal", + "bee eater", + "hornbill", + "hummingbird", + "jacamar", + "toucan", + "duck", + "red-breasted merganser", + "goose", + "black swan", + "tusker", + "echidna", + "platypus", + "wallaby", + "koala", + "wombat", + "jellyfish", + "sea anemone", + "brain coral", + "flatworm", + "nematode", + "conch", + "snail", + "slug", + "sea slug", + "chiton", + "chambered nautilus", + "Dungeness crab", + "rock crab", + "fiddler crab", + "red king crab", + "American lobster", + "spiny lobster", + "crayfish", + "hermit crab", + "isopod", + "white stork", + "black stork", + "spoonbill", + "flamingo", + "little blue heron", + "great egret", + "bittern", + "crane (bird)", + "limpkin", + "common gallinule", + "American coot", + "bustard", + "ruddy turnstone", + "dunlin", + "common redshank", + "dowitcher", + "oystercatcher", + "pelican", + "king penguin", + "albatross", + "grey whale", + "killer whale", + "dugong", + "sea lion", + "Chihuahua", + "Japanese Chin", + "Maltese", + "Pekingese", + "Shih Tzu", + "King Charles Spaniel", + "Papillon", + "toy terrier", + "Rhodesian Ridgeback", + "Afghan Hound", + "Basset Hound", + "Beagle", + "Bloodhound", + "Bluetick Coonhound", + "Black and Tan Coonhound", + "Treeing Walker Coonhound", + "English foxhound", + "Redbone Coonhound", + "borzoi", + "Irish Wolfhound", + "Italian Greyhound", + "Whippet", + "Ibizan Hound", + "Norwegian Elkhound", + "Otterhound", + "Saluki", + "Scottish Deerhound", + "Weimaraner", + "Staffordshire Bull Terrier", + "American Staffordshire Terrier", + "Bedlington Terrier", + "Border Terrier", + "Kerry Blue Terrier", + "Irish Terrier", + "Norfolk Terrier", + "Norwich Terrier", + "Yorkshire Terrier", + "Wire Fox Terrier", + "Lakeland Terrier", + "Sealyham Terrier", + "Airedale Terrier", + "Cairn Terrier", + "Australian Terrier", + "Dandie Dinmont Terrier", + "Boston Terrier", + "Miniature Schnauzer", + "Giant Schnauzer", + "Standard Schnauzer", + "Scottish Terrier", + "Tibetan Terrier", + "Australian Silky Terrier", + "Soft-coated Wheaten Terrier", + "West Highland White Terrier", + "Lhasa Apso", + "Flat-Coated Retriever", + "Curly-coated Retriever", + "Golden Retriever", + "Labrador Retriever", + "Chesapeake Bay Retriever", + "German Shorthaired Pointer", + "Vizsla", + "English Setter", + "Irish Setter", + "Gordon Setter", + "Brittany", + "Clumber Spaniel", + "English Springer Spaniel", + "Welsh Springer Spaniel", + "Cocker Spaniels", + "Sussex Spaniel", + "Irish Water Spaniel", + "Kuvasz", + "Schipperke", + "Groenendael", + "Malinois", + "Briard", + "Australian Kelpie", + "Komondor", + "Old English Sheepdog", + "Shetland Sheepdog", + "collie", + "Border Collie", + "Bouvier des Flandres", + "Rottweiler", + "German Shepherd Dog", + "Dobermann", + "Miniature Pinscher", + "Greater Swiss Mountain Dog", + "Bernese Mountain Dog", + "Appenzeller Sennenhund", + "Entlebucher Sennenhund", + "Boxer", + "Bullmastiff", + "Tibetan Mastiff", + "French Bulldog", + "Great Dane", + "St. Bernard", + "husky", + "Alaskan Malamute", + "Siberian Husky", + "Dalmatian", + "Affenpinscher", + "Basenji", + "pug", + "Leonberger", + "Newfoundland", + "Pyrenean Mountain Dog", + "Samoyed", + "Pomeranian", + "Chow Chow", + "Keeshond", + "Griffon Bruxellois", + "Pembroke Welsh Corgi", + "Cardigan Welsh Corgi", + "Toy Poodle", + "Miniature Poodle", + "Standard Poodle", + "Mexican hairless dog", + "grey wolf", + "Alaskan tundra wolf", + "red wolf", + "coyote", + "dingo", + "dhole", + "African wild dog", + "hyena", + "red fox", + "kit fox", + "Arctic fox", + "grey fox", + "tabby cat", + "tiger cat", + "Persian cat", + "Siamese cat", + "Egyptian Mau", + "cougar", + "lynx", + "leopard", + "snow leopard", + "jaguar", + "lion", + "tiger", + "cheetah", + "brown bear", + "American black bear", + "polar bear", + "sloth bear", + "mongoose", + "meerkat", + "tiger beetle", + "ladybug", + "ground beetle", + "longhorn beetle", + "leaf beetle", + "dung beetle", + "rhinoceros beetle", + "weevil", + "fly", + "bee", + "ant", + "grasshopper", + "cricket", + "stick insect", + "cockroach", + "mantis", + "cicada", + "leafhopper", + "lacewing", + "dragonfly", + "damselfly", + "red admiral", + "ringlet", + "monarch butterfly", + "small white", + "sulphur butterfly", + "gossamer-winged butterfly", + "starfish", + "sea urchin", + "sea cucumber", + "cottontail rabbit", + "hare", + "Angora rabbit", + "hamster", + "porcupine", + "fox squirrel", + "marmot", + "beaver", + "guinea pig", + "common sorrel", + "zebra", + "pig", + "wild boar", + "warthog", + "hippopotamus", + "ox", + "water buffalo", + "bison", + "ram", + "bighorn sheep", + "Alpine ibex", + "hartebeest", + "impala", + "gazelle", + "dromedary", + "llama", + "weasel", + "mink", + "European polecat", + "black-footed ferret", + "otter", + "skunk", + "badger", + "armadillo", + "three-toed sloth", + "orangutan", + "gorilla", + "chimpanzee", + "gibbon", + "siamang", + "guenon", + "patas monkey", + "baboon", + "macaque", + "langur", + "black-and-white colobus", + "proboscis monkey", + "marmoset", + "white-headed capuchin", + "howler monkey", + "titi", + "Geoffroy's spider monkey", + "common squirrel monkey", + "ring-tailed lemur", + "indri", + "Asian elephant", + "African bush elephant", + "red panda", + "giant panda", + "snoek", + "eel", + "coho salmon", + "rock beauty", + "clownfish", + "sturgeon", + "garfish", + "lionfish", + "pufferfish", + "abacus", + "abaya", + "academic gown", + "accordion", + "acoustic guitar", + "aircraft carrier", + "airliner", + "airship", + "altar", + "ambulance", + "amphibious vehicle", + "analog clock", + "apiary", + "apron", + "waste container", + "assault rifle", + "backpack", + "bakery", + "balance beam", + "balloon", + "ballpoint pen", + "Band-Aid", + "banjo", + "baluster", + "barbell", + "barber chair", + "barbershop", + "barn", + "barometer", + "barrel", + "wheelbarrow", + "baseball", + "basketball", + "bassinet", + "bassoon", + "swimming cap", + "bath towel", + "bathtub", + "station wagon", + "lighthouse", + "beaker", + "military cap", + "beer bottle", + "beer glass", + "bell-cot", + "bib", + "tandem bicycle", + "bikini", + "ring binder", + "binoculars", + "birdhouse", + "boathouse", + "bobsleigh", + "bolo tie", + "poke bonnet", + "bookcase", + "bookstore", + "bottle cap", + "bow", + "bow tie", + "brass", + "bra", + "breakwater", + "breastplate", + "broom", + "bucket", + "buckle", + "bulletproof vest", + "high-speed train", + "butcher shop", + "taxicab", + "cauldron", + "candle", + "cannon", + "canoe", + "can opener", + "cardigan", + "car mirror", + "carousel", + "tool kit", + "carton", + "car wheel", + "automated teller machine", + "cassette", + "cassette player", + "castle", + "catamaran", + "CD player", + "cello", + "mobile phone", + "chain", + "chain-link fence", + "chain mail", + "chainsaw", + "chest", + "chiffonier", + "chime", + "china cabinet", + "Christmas stocking", + "church", + "movie theater", + "cleaver", + "cliff dwelling", + "cloak", + "clogs", + "cocktail shaker", + "coffee mug", + "coffeemaker", + "coil", + "combination lock", + "computer keyboard", + "confectionery store", + "container ship", + "convertible", + "corkscrew", + "cornet", + "cowboy boot", + "cowboy hat", + "cradle", + "crane (machine)", + "crash helmet", + "crate", + "infant bed", + "Crock Pot", + "croquet ball", + "crutch", + "cuirass", + "dam", + "desk", + "desktop computer", + "rotary dial telephone", + "diaper", + "digital clock", + "digital watch", + "dining table", + "dishcloth", + "dishwasher", + "disc brake", + "dock", + "dog sled", + "dome", + "doormat", + "drilling rig", + "drum", + "drumstick", + "dumbbell", + "Dutch oven", + "electric fan", + "electric guitar", + "electric locomotive", + "entertainment center", + "envelope", + "espresso machine", + "face powder", + "feather boa", + "filing cabinet", + "fireboat", + "fire engine", + "fire screen sheet", + "flagpole", + "flute", + "folding chair", + "football helmet", + "forklift", + "fountain", + "fountain pen", + "four-poster bed", + "freight car", + "French horn", + "frying pan", + "fur coat", + "garbage truck", + "gas mask", + "gas pump", + "goblet", + "go-kart", + "golf ball", + "golf cart", + "gondola", + "gong", + "gown", + "grand piano", + "greenhouse", + "grille", + "grocery store", + "guillotine", + "barrette", + "hair spray", + "half-track", + "hammer", + "hamper", + "hair dryer", + "hand-held computer", + "handkerchief", + "hard disk drive", + "harmonica", + "harp", + "harvester", + "hatchet", + "holster", + "home theater", + "honeycomb", + "hook", + "hoop skirt", + "horizontal bar", + "horse-drawn vehicle", + "hourglass", + "iPod", + "clothes iron", + "jack-o'-lantern", + "jeans", + "jeep", + "T-shirt", + "jigsaw puzzle", + "pulled rickshaw", + "joystick", + "kimono", + "knee pad", + "knot", + "lab coat", + "ladle", + "lampshade", + "laptop computer", + "lawn mower", + "lens cap", + "paper knife", + "library", + "lifeboat", + "lighter", + "limousine", + "ocean liner", + "lipstick", + "slip-on shoe", + "lotion", + "speaker", + "loupe", + "sawmill", + "magnetic compass", + "mail bag", + "mailbox", + "tights", + "tank suit", + "manhole cover", + "maraca", + "marimba", + "mask", + "match", + "maypole", + "maze", + "measuring cup", + "medicine chest", + "megalith", + "microphone", + "microwave oven", + "military uniform", + "milk can", + "minibus", + "miniskirt", + "minivan", + "missile", + "mitten", + "mixing bowl", + "mobile home", + "Model T", + "modem", + "monastery", + "monitor", + "moped", + "mortar", + "square academic cap", + "mosque", + "mosquito net", + "scooter", + "mountain bike", + "tent", + "computer mouse", + "mousetrap", + "moving van", + "muzzle", + "nail", + "neck brace", + "necklace", + "nipple", + "notebook computer", + "obelisk", + "oboe", + "ocarina", + "odometer", + "oil filter", + "organ", + "oscilloscope", + "overskirt", + "bullock cart", + "oxygen mask", + "packet", + "paddle", + "paddle wheel", + "padlock", + "paintbrush", + "pajamas", + "palace", + "pan flute", + "paper towel", + "parachute", + "parallel bars", + "park bench", + "parking meter", + "passenger car", + "patio", + "payphone", + "pedestal", + "pencil case", + "pencil sharpener", + "perfume", + "Petri dish", + "photocopier", + "plectrum", + "Pickelhaube", + "picket fence", + "pickup truck", + "pier", + "piggy bank", + "pill bottle", + "pillow", + "ping-pong ball", + "pinwheel", + "pirate ship", + "pitcher", + "hand plane", + "planetarium", + "plastic bag", + "plate rack", + "plow", + "plunger", + "Polaroid camera", + "pole", + "police van", + "poncho", + "billiard table", + "soda bottle", + "pot", + "potter's wheel", + "power drill", + "prayer rug", + "printer", + "prison", + "projectile", + "projector", + "hockey puck", + "punching bag", + "purse", + "quill", + "quilt", + "race car", + "racket", + "radiator", + "radio", + "radio telescope", + "rain barrel", + "recreational vehicle", + "reel", + "reflex camera", + "refrigerator", + "remote control", + "restaurant", + "revolver", + "rifle", + "rocking chair", + "rotisserie", + "eraser", + "rugby ball", + "ruler", + "running shoe", + "safe", + "safety pin", + "salt shaker", + "sandal", + "sarong", + "saxophone", + "scabbard", + "weighing scale", + "school bus", + "schooner", + "scoreboard", + "CRT screen", + "screw", + "screwdriver", + "seat belt", + "sewing machine", + "shield", + "shoe store", + "shoji", + "shopping basket", + "shopping cart", + "shovel", + "shower cap", + "shower curtain", + "ski", + "ski mask", + "sleeping bag", + "slide rule", + "sliding door", + "slot machine", + "snorkel", + "snowmobile", + "snowplow", + "soap dispenser", + "soccer ball", + "sock", + "solar thermal collector", + "sombrero", + "soup bowl", + "space bar", + "space heater", + "space shuttle", + "spatula", + "motorboat", + "spider web", + "spindle", + "sports car", + "spotlight", + "stage", + "steam locomotive", + "through arch bridge", + "steel drum", + "stethoscope", + "scarf", + "stone wall", + "stopwatch", + "stove", + "strainer", + "tram", + "stretcher", + "couch", + "stupa", + "submarine", + "suit", + "sundial", + "sunglass", + "sunglasses", + "sunscreen", + "suspension bridge", + "mop", + "sweatshirt", + "swimsuit", + "swing", + "switch", + "syringe", + "table lamp", + "tank", + "tape player", + "teapot", + "teddy bear", + "television", + "tennis ball", + "thatched roof", + "front curtain", + "thimble", + "threshing machine", + "throne", + "tile roof", + "toaster", + "tobacco shop", + "toilet seat", + "torch", + "totem pole", + "tow truck", + "toy store", + "tractor", + "semi-trailer truck", + "tray", + "trench coat", + "tricycle", + "trimaran", + "tripod", + "triumphal arch", + "trolleybus", + "trombone", + "tub", + "turnstile", + "typewriter keyboard", + "umbrella", + "unicycle", + "upright piano", + "vacuum cleaner", + "vase", + "vault", + "velvet", + "vending machine", + "vestment", + "viaduct", + "violin", + "volleyball", + "waffle iron", + "wall clock", + "wallet", + "wardrobe", + "military aircraft", + "sink", + "washing machine", + "water bottle", + "water jug", + "water tower", + "whiskey jug", + "whistle", + "wig", + "window screen", + "window shade", + "Windsor tie", + "wine bottle", + "wing", + "wok", + "wooden spoon", + "wool", + "split-rail fence", + "shipwreck", + "yawl", + "yurt", + "website", + "comic book", + "crossword", + "traffic sign", + "traffic light", + "dust jacket", + "menu", + "plate", + "guacamole", + "consomme", + "hot pot", + "trifle", + "ice cream", + "ice pop", + "baguette", + "bagel", + "pretzel", + "cheeseburger", + "hot dog", + "mashed potato", + "cabbage", + "broccoli", + "cauliflower", + "zucchini", + "spaghetti squash", + "acorn squash", + "butternut squash", + "cucumber", + "artichoke", + "bell pepper", + "cardoon", + "mushroom", + "Granny Smith", + "strawberry", + "orange", + "lemon", + "fig", + "pineapple", + "banana", + "jackfruit", + "custard apple", + "pomegranate", + "hay", + "carbonara", + "chocolate syrup", + "dough", + "meatloaf", + "pizza", + "pot pie", + "burrito", + "red wine", + "espresso", + "cup", + "eggnog", + "alp", + "bubble", + "cliff", + "coral reef", + "geyser", + "lakeshore", + "promontory", + "shoal", + "seashore", + "valley", + "volcano", + "baseball player", + "bridegroom", + "scuba diver", + "rapeseed", + "daisy", + "yellow lady's slipper", + "corn", + "acorn", + "rose hip", + "horse chestnut seed", + "coral fungus", + "agaric", + "gyromitra", + "stinkhorn mushroom", + "earth star", + "hen-of-the-woods", + "bolete", + "ear", + "toilet paper", +]; + +const voc_detection_labels = [ + 'aeroplane', + 'bicycle', + 'bird', + 'boat', + 'bottle', + 'bus', + 'car', + 'cat', + 'chair', + 'cow', + 'diningtable', + 'dog', + 'horse', + 'motorbike', + 'person', + 'pottedplant', + 'sheep', + 'sofa', + 'train', + 'tvmonitor', +]; + +const coco_detection_labels = [ + 'person', + 'bicycle', + 'car', + 'motorcycle', + 'airplane', + 'bus', + 'train', + 'truck', + 'boat', + 'traffic light', + 'fire hydrant', + 'stop sign', + 'parking meter', + 'bench', 'bird', + 'cat', + 'dog', + 'horse', + 'sheep', + 'cow', + 'elephant', + 'bear', + 'zebra', + 'giraffe', + 'backpack', + 'umbrella', + 'handbag', + 'tie', + 'suitcase', + 'frisbee', + 'skis', + 'snowboard', + 'sports ball', + 'kite', + 'baseball bat', + 'baseball glove', + 'skateboard', + 'surfboard', + 'tennis racket', + 'bottle', + 'wine glass', + 'cup', + 'fork', + 'knife', + 'spoon', + 'bowl', + 'banana', + 'apple', + 'sandwich', + 'orange', + 'broccoli', + 'carrot', + 'hot dog', + 'pizza', + 'donut', + 'cake', + 'chair', + 'couch', + 'potted plant', + 'bed', + 'dining table', + 'toilet', + 'tv', + 'laptop', + 'mouse', + 'remote', + 'keyboard', + 'cell phone', + 'microwave', + 'oven', + 'toaster', + 'sink', + 'refrigerator', + 'book', + 'clock', + 'vase', + 'scissors', + 'teddy bear', + 'hair drier', + 'toothbrush' +] diff --git a/docs/_static/js/main.js b/docs/_static/js/main.js new file mode 100644 index 0000000000..a9510d5828 --- /dev/null +++ b/docs/_static/js/main.js @@ -0,0 +1,303 @@ +const tasks = { + CLASSIFICATION: 'classification', + OBJECT_DETECTION: 'object_detection', + SEMANTIC_SEGMENTATION: 'semantic_segmentation', + INSTANCE_SEGMENTATION: 'instance_segmentation', + POSE_ESTIMATION: 'pose_estimation', +} + +var media_constraints = { + video: { + width: 720, + height: 480, + }, + audio: false, +}; +var current_stream; + +var model_path; +var task; +var model; +var classification_session; +var detection_session; +var postprocessor; +const preprocessor = new Preprocessor(); + +function block_ui_on_loading() { + const predict_button = document.getElementById('predict_button'); + const classification_tab = document.getElementById('classification_tab'); + const object_detection_tab = document.getElementById('object_detection_tab'); + predict_button.innerHTML = 'Loading'; + predict_button.disabled = true; + classification_tab.disabled = true; + object_detection_tab.disabled = true; +} + +function unblock_ui_on_loading() { + const predict_button = document.getElementById('predict_button'); + const classification_tab = document.getElementById('classification_tab'); + const object_detection_tab = document.getElementById('object_detection_tab'); + predict_button.innerHTML = 'Predict'; + predict_button.disabled = false; + classification_tab.disabled = false; + object_detection_tab.disabled = false; +} + +async function on_classification() { + if (task == tasks.CLASSIFICATION) { return; } + processor.clear(); + block_ui_on_loading(); + model_path = 'https://apache-mxnet.s3-us-west-2.amazonaws.com/onnx/models/gluoncv-mobilenetv3_large-ad683fdc.onnx'; + // model_path = 'https://damp-mountain-14992.herokuapp.com/https://apache-mxnet.s3-us-west-2.amazonaws.com/onnx/models/gluoncv-mobilenetv3_large-ad683fdc.onnx'; + // model_path = '_static/models/mobilenetv3_large.onnx' //local test only + task = tasks.CLASSIFICATION; + model = new Model(model_path, 224, 224, task, image_net_labels); + postprocessor = new Postprocessor(model.task); + // the website won't cache the model for some reason, resulting redownload each time + if (classification_session === undefined) { + await ort.InferenceSession.create(model.path).then((session) => { + classification_session = session; + unblock_ui_on_loading(); + }); + } else { + unblock_ui_on_loading(); + } +} + +async function on_obj_detection() { + if (task == tasks.OBJECT_DETECTION) { return; } + processor.clear(); + block_ui_on_loading(); + model_path = 'https://apache-mxnet.s3-us-west-2.amazonaws.com/onnx/models/gluoncv-yolo3_mobilenet1.0_coco-115299e3.onnx'; + // model_path = 'https://damp-mountain-14992.herokuapp.com/https://apache-mxnet.s3-us-west-2.amazonaws.com/onnx/models/gluoncv-yolo3_mobilenet1.0_coco-115299e3.onnx'; + // model_path = '_static/models/yolo3_mobilenet1.0_voc.onnx' //local test only + task = tasks.OBJECT_DETECTION; + model = new Model(model_path, 512, 512, task, coco_detection_labels); + postprocessor = new Postprocessor(model.task); + // the website won't cache the model for some reason, resulting redownload each time + if (detection_session === undefined) { + await ort.InferenceSession.create(model.path).then((session) => { + detection_session = session; + unblock_ui_on_loading(); + }); + } else { + unblock_ui_on_loading(); + } +} + +function predict() { + processor.computeFrame(); +} + +var processor = { + did_load: false, + + do_load: function() { + this.video = document.getElementById('local_video_stream'); + this.video_width = this.video.width; + this.video_height = this.video.height; + this.canvas = document.getElementById('result_canvas'); + this.canvas_ctx = this.canvas.getContext('2d', { alpha: false }); + this.did_load = true; + }, + + show_classification_result: function(label, prob) { + var classification_left_element = document.getElementById('classification_left'); + var classification_right_element = document.getElementById('classification_right'); + + var label_element = document.createElement('p'); + label_element.innerHTML = label; + label_element.className = 'classification_label'; + classification_left_element.appendChild(label_element); + + var div = document.createElement('div'); + var prob_bar_element = document.createElement('progress'); + prob_bar_element.className = 'classification_prob_bar'; + prob_bar_element.value = prob; + prob_bar_element.max = '100'; + var prob_element = document.createElement('p'); + prob_element.innerHTML = prob + '%'; + prob_element.className = 'classification_prob'; + + div.appendChild(prob_bar_element); + div.appendChild(prob_element); + classification_right_element.appendChild(div); + }, + + draw_bbox(label, score, bbox, color) { + [xmin, ymin, width, height] = bbox; + this.canvas_ctx.strokeStyle = color; + this.canvas_ctx.fillStyle = color; + this.canvas_ctx.lineWidth = 3; + this.canvas_ctx.font = "30px Comic Sans MS"; + this.canvas_ctx.strokeRect(xmin, ymin, width, height); + if (ymin-30 <= 0) { + this.canvas_ctx.fillText(`${label} ${score}`, xmin, ymin+30); + } else { + this.canvas_ctx.fillText(`${label} ${score}`, xmin, ymin-10); + } + }, + + visualize: function(processed_result) { + var classification_results_element = document.getElementById('classification_results'); + classification_results_element.style.visibility = 'hidden'; + switch (model.task) { + case tasks.CLASSIFICATION: + var classification_results_element = document.getElementById('classification_results'); + var classification_left_element = document.getElementById('classification_left'); + var classification_right_element = document.getElementById('classification_right'); + classification_left_element.innerHTML = ''; + classification_right_element.innerHTML = ''; + classification_results_element.style.visibility = 'visible'; + processed_result.map((r) => this.show_classification_result(r.label, r.prob)); + break; + case tasks.OBJECT_DETECTION: + [classes, scores, bboxes, color_maps] = processed_result; + for (var i = 0; i < classes.length; i++) { + this.draw_bbox(classes[i], scores[i], bboxes[i], color_maps[classes[i]]); + } + break; + default: + alert('Error: task ' + model.task + ' has not been implemented'); + break; + } + }, + + computeFrame: async function() { + if (this.video.paused || this.video.ended) { + return; + } + this.canvas_ctx.drawImage(this.video, 0, 0, model.input_width, model.input_height); + var frame = this.canvas_ctx.getImageData(0, 0, model.input_width, model.input_height); + this.canvas_ctx.drawImage(this.video, 0, 0, this.video_width, this.video_height); + var frame_length = frame.data.length / 4; + var rgba_frame_f32 = Float32Array.from(frame.data); + var rgb_frame_f32 = preprocessor.remove_alpha_channel(rgba_frame_f32, frame_length); + + const image_tensor = new ort.Tensor('float32', rgb_frame_f32, [1,model.input_width,model.input_height,3]); + var result; + var data; + // extract the data from result and visualize + switch (model.task) { + case tasks.CLASSIFICATION: + result = await classification_session.run({data: image_tensor}); + data = Object.keys(result).map((key) => result[key])[0].data; + this.visualize(postprocessor.process(data, { k:5 })); + break; + case tasks.OBJECT_DETECTION: + result = await detection_session.run({data: image_tensor}); + data = Object.keys(result).map((key) => result[key].data); + this.visualize(postprocessor.process(data, { + video_width: this.video_width, + video_height: this.video_height, + input_width: model.input_width, + input_height: model.input_height, + threshold: 0.5, + } + )); + + break; + default: + alert('Error: task ' + model.task + ' has not been implemented'); + break; + } + + return; + }, + + clear: function() { + if (this.canvas_ctx) { + this.canvas_ctx.clearRect(0, 0, this.video_width, this.video_height); + this.canvas_ctx.rect(0, 0, this.video_width, this.video_height); + this.canvas_ctx.fillStyle = 'black'; + this.canvas_ctx.fill(); + } + var classification_results_element = document.getElementById('classification_results'); + classification_results_element.style.visibility = 'hidden'; + } +}; + +function got_devices(mediaDevices) { + const camera_select = document.getElementById('camera_select'); + camera_select.innerHTML = ''; + let count = 1; + mediaDevices.forEach(mediaDevice => { + if (mediaDevice.kind === 'videoinput') { + const option = document.createElement('option'); + option.value = mediaDevice.deviceId; + const label = mediaDevice.label || `Camera ${count++}`; + const textNode = document.createTextNode(label); + option.appendChild(textNode); + camera_select.appendChild(option); + } + }); +} + +function prepare_devices() { + navigator.mediaDevices.enumerateDevices() + .then(got_devices) + .then(get_camera); +} + +function get_camera() { + if (typeof current_stream !== undefined) { + stop_current_stream(); + } + const camera_select = document.getElementById('camera_select'); + if (camera_select.value === '') { + media_constraints.video.facingMode = 'environment'; + } else { + media_constraints.video.deviceId = { exact: camera_select.value }; + } + navigator.mediaDevices.getUserMedia(media_constraints) + .then(function (stream) { + var local_video_stream = document.getElementById('local_video_stream'); + local_video_stream.srcObject = stream; + processor.do_load(); + current_stream = stream; + }) + .then(()=> { + document.getElementById('video_container').style.visibility = 'visible'; + }) + .catch(handle_get_user_media_error); +} + +function stop_current_stream() { + if (current_stream) { + current_stream.getTracks().forEach(track => { + track.stop(); + }); + } +} + +function handle_get_user_media_error(e) { + switch(e.name) { + case 'NotFoundError': + alert('Unable to push video because no camera was found'); + break; + case 'SecurityError': + case 'PermissionDeniedError': + break; + default: + alert('Error opening your camera: ' + e.message); + break; + } +} + +async function main() { + try { + prepare_devices(); + } catch (e) { + alert(e); + } + +} + +$(document).ready(function() { + $('.tab button').on('click', function() { + $('.tab button').removeClass('selected'); + $(this).addClass('selected'); + }); + document.getElementById('classification_tab').click(); // provide a default tab + main(); +}); diff --git a/docs/_static/js/model.js b/docs/_static/js/model.js new file mode 100644 index 0000000000..a964f5efbb --- /dev/null +++ b/docs/_static/js/model.js @@ -0,0 +1,9 @@ +class Model { + constructor(model_path, model_input_width, model_input_height, model_task, classes) { + this.path = model_path; + this.input_width = model_input_width; + this.input_height = model_input_height; + this.task = model_task; + this.classes = classes; + } +} diff --git a/docs/_static/js/postprocessor.js b/docs/_static/js/postprocessor.js new file mode 100644 index 0000000000..95cb5056a9 --- /dev/null +++ b/docs/_static/js/postprocessor.js @@ -0,0 +1,129 @@ +class Postprocessor { + constructor(task) { + this.task = task; + this.color_maps = {}; + } + + get_random_color() { + var letters = '0123456789ABCDEF'; + var color = '#'; + for (var i = 0; i < 6; i++) { + color += letters[Math.floor(Math.random() * 16)]; + } + return color; + } + + scores_to_probs(scores) { + var probs = []; + const softmax_sum = scores.reduce((a, b) => a + Math.exp(b), 0); + scores.forEach((score) => probs.push(Math.exp(score) / softmax_sum)); + return probs; + } + + process_classification(scores, ctx) { + var k = ctx.k; + if (!k) { k = 5; } + const probs = this.scores_to_probs(Array.from(scores)); + const probs_indices = probs.map( + function (prob, index) { + return [prob, index]; + } + ); + const sorted = probs_indices.sort( + function (a, b) { + if (a[0] < b[0]) { + return -1; + } + if (a[0] > b[0]) { + return 1; + } + return 0; + } + ).reverse(); + const topK = sorted.slice(0, k).map(function (prob_index) { + const i_class = model.classes[prob_index[1]]; + return { + label: i_class, + prob: (prob_index[0] * 100).toFixed(2), + index: prob_index[1], + }; + }); + return topK; + } + + remap_bbox(bbox, video_width, video_height, input_width, input_height) { + const xmin = bbox[0]; + const ymin = bbox[1]; + const xmax = bbox[2]; + const ymax = bbox[3]; + const new_xmin = Math.max(Math.round(xmin * (video_width / input_width)), 0); + const new_ymin = Math.max(Math.round(ymin * (video_height / input_height)), 0); + const new_xmax = Math.min(Math.round(xmax * (video_width / input_width)), video_width); + const new_ymax = Math.min(Math.round(ymax * (video_height / input_height)), video_height); + const new_bbox_width = new_xmax - new_xmin; + const new_bbox_height = new_ymax - new_ymin; + + // js requires starting point, width, height to draw rect + return [new_xmin, new_ymin, new_bbox_width, new_bbox_height]; + } + + process_object_detection(result, ctx) { + const class_ids = result[0]; + const scores = result[1]; + const bboxes = result[2]; + if ( class_ids.length != bboxes.length/4) { + throw `The length of labels and bboxes must match ${class_ids.length} vs ${bboxes.length/4}`; + } + if (scores.length != bboxes.length/4) { + throw `The length of scores and bboxes must match ${scores.length} vs ${bboxes.length/4}`; + } + const num_result = class_ids.length; + var video_width = ctx.video_width; + var video_height = ctx.video_height; + var input_width = ctx.input_width; + var input_height = ctx.input_height; + var threshold = ctx.threshold; + if (!video_width) { video_width = 512; } + if (!video_height) { video_height = 512; } + if (!threshold) { threshold = 0.5; } + + var processed_class_ids = []; + var processed_scores = []; + var processed_bboxes = []; + + for (var i = 0; i < num_result; i++) { + if (scores[i] < threshold) { continue; } + if (class_ids[i] < 0) { continue; } + const bbox = [ + Math.round(bboxes[i*4+0]), + Math.round(bboxes[i*4+1]), + Math.round(bboxes[i*4+2]), + Math.round(bboxes[i*4+3]) + ]; + processed_class_ids.push(model.classes[class_ids[i]]); + processed_scores.push(scores[i].toFixed(3)); + processed_bboxes.push(this.remap_bbox(bbox, video_width, video_height, input_width, input_height)); + if (!(model.classes[class_ids[i]] in this.color_maps)) { + this.color_maps[model.classes[class_ids[i]]] = this.get_random_color(); + } + } + + return [processed_class_ids, processed_scores, processed_bboxes, this.color_maps]; + + } + + process(result, ctx) { + switch (this.task) { + case tasks.CLASSIFICATION: + return this.process_classification(result, ctx); + case tasks.OBJECT_DETECTION: + return this.process_object_detection(result, ctx); + case tasks.SEMANTIC_SEGMENTATION: + break; + case tasksk.INSTANCE_SEGMENTATION: + break; + case ttasks.POSE_ESTIMATION: + break; + } + } +} diff --git a/docs/_static/js/preprocessor.js b/docs/_static/js/preprocessor.js new file mode 100644 index 0000000000..3c3d97b4d6 --- /dev/null +++ b/docs/_static/js/preprocessor.js @@ -0,0 +1,33 @@ +class Preprocessor { + constructor() { + this.norm_val = { + r_mean: 0.485, + g_mean: 0.456, + b_mean: 0.406, + r_std: 0.229, + g_std: 0.224, + b_std: 0.225 + }; + } + + remove_alpha_channel(rgba_f32, length) { + var rgb_frame_f32 = new Float32Array(length * 3); + for (var i = 0; i < length; i++) { + rgb_frame_f32[i * 3 + 0] = rgba_f32[i * 4 + 0]; + rgb_frame_f32[i * 3 + 1] = rgba_f32[i * 4 + 1]; + rgb_frame_f32[i * 3 + 2] = rgba_f32[i * 4 + 2]; + } + return rgb_frame_f32; + } + + normalize(rgb_frame_f32, l) { + for (var i = 0; i < l; i++) { + var r = rgb_frame_f32[i * 3 + 0]; + var g = rgb_frame_f32[i * 3 + 1]; + var b = rgb_frame_f32[i * 3 + 2]; + rgb_frame_f32[i * 3 + 0] = ((r / 255) - this.norm_val.r_mean) / this.norm_val.r_std; + rgb_frame_f32[i * 3 + 1] = ((g / 255) - this.norm_val.g_mean) / this.norm_val.g_std; + rgb_frame_f32[i * 3 + 2] = ((b / 255) - this.norm_val.b_mean) / this.norm_val.b_std; + } + } +} diff --git a/docs/_templates/demo.html b/docs/_templates/demo.html new file mode 100644 index 0000000000..220e3a49dc --- /dev/null +++ b/docs/_templates/demo.html @@ -0,0 +1,437 @@ + + + + + + + + + + + GluonCV Quick Demo + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + GluonCV + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+
+

Quick Demo

+ +
+
+ +
+
+
+
+
+
+ + + + + + + +
+ + + + diff --git a/docs/_templates/index.html b/docs/_templates/index.html index 22d3d593f0..6ca28a7421 100644 --- a/docs/_templates/index.html +++ b/docs/_templates/index.html @@ -241,6 +241,7 @@
+ +
+
+
+
+
+

Quick Demo

+ Check it out +
+
+
+
+
+
+
diff --git a/docs/conf.py b/docs/conf.py index 7b692bfc6d..87ae298363 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -132,7 +132,7 @@ master_doc = 'contents' # Use customize landing page -html_additional_pages = {'index': 'index.html'} +html_additional_pages = {'index': 'index.html', 'demo': 'demo.html'} # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages.