{ "cells": [ { "cell_type": "code", "execution_count": 3, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1000 }, "colab_type": "code", "id": "1ph7Fqve7VXD", "outputId": "59484e0a-cd5b-40fb-c08f-2ae6d9b46136" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting tensorflow-gpu<2\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/98/ab/19aba3629427c2d96790f73838639136ce02b6e7e1c4f2dd60149174c794/tensorflow_gpu-1.15.3-cp36-cp36m-manylinux2010_x86_64.whl (411.0MB)\n", "\u001b[K |████████████████████████████████| 411.0MB 37kB/s \n", "\u001b[?25hCollecting dm-sonnet<2\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/53/14/e221b910127bf4e2c19bc6d3b3e65a4e0104b90f7e98a3d428926474ece3/dm_sonnet-1.36-py3-none-any.whl (665kB)\n", "\u001b[K |████████████████████████████████| 665kB 37.2MB/s \n", "\u001b[?25hCollecting tensorflow-probability==0.7.0\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/3e/3a/c10b6c22320531c774402ac7186d1b673374e2a9d12502cbc8d811e4601c/tensorflow_probability-0.7.0-py2.py3-none-any.whl (981kB)\n", "\u001b[K |████████████████████████████████| 983kB 42.4MB/s \n", "\u001b[?25hCollecting gast==0.2.2\n", " Downloading https://files.pythonhosted.org/packages/4e/35/11749bf99b2d4e3cceb4d55ca22590b0d7c2c62b9de38ac4a4a7f4687421/gast-0.2.2.tar.gz\n", "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu<2) (3.10.0)\n", "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu<2) (0.8.1)\n", "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu<2) (3.2.1)\n", "Requirement already satisfied: keras-applications>=1.0.8 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu<2) (1.0.8)\n", "Collecting tensorflow-estimator==1.15.1\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/de/62/2ee9cd74c9fa2fa450877847ba560b260f5d0fb70ee0595203082dafcc9d/tensorflow_estimator-1.15.1-py2.py3-none-any.whl (503kB)\n", "\u001b[K |████████████████████████████████| 512kB 48.9MB/s \n", "\u001b[?25hRequirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu<2) (1.1.0)\n", "Requirement already satisfied: wheel>=0.26; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu<2) (0.34.2)\n", "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu<2) (1.1.2)\n", "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu<2) (1.12.0)\n", "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu<2) (1.12.1)\n", "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu<2) (1.29.0)\n", "Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu<2) (0.2.0)\n", "Collecting tensorboard<1.16.0,>=1.15.0\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/1e/e9/d3d747a97f7188f48aa5eda486907f3b345cd409f0a0850468ba867db246/tensorboard-1.15.0-py3-none-any.whl (3.8MB)\n", "\u001b[K |████████████████████████████████| 3.8MB 55.3MB/s \n", "\u001b[?25hRequirement already satisfied: numpy<2.0,>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu<2) (1.18.5)\n", "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu<2) (0.9.0)\n", "Requirement already satisfied: contextlib2 in /usr/local/lib/python3.6/dist-packages (from dm-sonnet<2) (0.5.5)\n", "Collecting semantic-version\n", " Downloading https://files.pythonhosted.org/packages/a5/15/00ef3b7888a10363b7c402350eda3acf395ff05bebae312d1296e528516a/semantic_version-2.8.5-py2.py3-none-any.whl\n", "Requirement already satisfied: cloudpickle>=0.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-probability==0.7.0) (1.3.0)\n", "Requirement already satisfied: decorator in /usr/local/lib/python3.6/dist-packages (from tensorflow-probability==0.7.0) (4.4.2)\n", "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow-gpu<2) (47.3.1)\n", "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.8->tensorflow-gpu<2) (2.10.0)\n", "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow-gpu<2) (1.0.1)\n", "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow-gpu<2) (3.2.2)\n", "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from markdown>=2.6.8->tensorboard<1.16.0,>=1.15.0->tensorflow-gpu<2) (1.6.1)\n", "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard<1.16.0,>=1.15.0->tensorflow-gpu<2) (3.1.0)\n", "Building wheels for collected packages: gast\n", " Building wheel for gast (setup.py) ... \u001b[?25l\u001b[?25hdone\n", " Created wheel for gast: filename=gast-0.2.2-cp36-none-any.whl size=7540 sha256=526ca7008bd8b9af68166c48116931875346d7eedebf28dae64eed4cc876446c\n", " Stored in directory: /root/.cache/pip/wheels/5c/2e/7e/a1d4d4fcebe6c381f378ce7743a3ced3699feb89bcfbdadadd\n", "Successfully built gast\n", "\u001b[31mERROR: tensorflow 2.2.0 has requirement gast==0.3.3, but you'll have gast 0.2.2 which is incompatible.\u001b[0m\n", "\u001b[31mERROR: tensorflow 2.2.0 has requirement tensorboard<2.3.0,>=2.2.0, but you'll have tensorboard 1.15.0 which is incompatible.\u001b[0m\n", "\u001b[31mERROR: tensorflow 2.2.0 has requirement tensorflow-estimator<2.3.0,>=2.2.0, but you'll have tensorflow-estimator 1.15.1 which is incompatible.\u001b[0m\n", "\u001b[31mERROR: dm-sonnet 1.36 has requirement tensorflow-probability<0.9.0,>=0.8.0, but you'll have tensorflow-probability 0.7.0 which is incompatible.\u001b[0m\n", "Installing collected packages: gast, tensorflow-estimator, tensorboard, tensorflow-gpu, tensorflow-probability, semantic-version, dm-sonnet\n", " Found existing installation: gast 0.3.3\n", " Uninstalling gast-0.3.3:\n", " Successfully uninstalled gast-0.3.3\n", " Found existing installation: tensorflow-estimator 2.2.0\n", " Uninstalling tensorflow-estimator-2.2.0:\n", " Successfully uninstalled tensorflow-estimator-2.2.0\n", " Found existing installation: tensorboard 2.2.2\n", " Uninstalling tensorboard-2.2.2:\n", " Successfully uninstalled tensorboard-2.2.2\n", " Found existing installation: tensorflow-probability 0.10.0\n", " Uninstalling tensorflow-probability-0.10.0:\n", " Successfully uninstalled tensorflow-probability-0.10.0\n", "Successfully installed dm-sonnet-1.36 gast-0.2.2 semantic-version-2.8.5 tensorboard-1.15.0 tensorflow-estimator-1.15.1 tensorflow-gpu-1.15.3 tensorflow-probability-0.7.0\n" ] }, { "data": { "application/vnd.colab-display-data+json": { "pip_warning": { "packages": [ "gast", "tensorboard", "tensorflow" ] } } }, "metadata": { "tags": [] }, "output_type": "display_data" } ], "source": [ "!pip install \"tensorflow-gpu<2\" \"dm-sonnet<2\" \"tensorflow-probability==0.7.0\"" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 139 }, "colab_type": "code", "id": "h0c31VoL7FuB", "outputId": "53cd330e-637e-4bd9-ead7-b19b427518a6" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Imageio: 'ffmpeg-linux64-v3.3.1' was not found on your computer; downloading it now.\n", "Try 1. Download from https://github.com/imageio/imageio-binaries/raw/master/ffmpeg/ffmpeg-linux64-v3.3.1 (43.8 MB)\n", "Downloading: 8192/45929032 bytes (0.0%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b999424/45929032 bytes (2.2%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b3039232/45929032 bytes (6.6%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b5881856/45929032 bytes (12.8%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b9682944/45929032 bytes (21.1%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b13500416/45929032 bytes (29.4%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b17342464/45929032 bytes (37.8%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b21094400/45929032 bytes (45.9%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b25034752/45929032 bytes (54.5%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b28819456/45929032 bytes (62.7%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b32768000/45929032 bytes (71.3%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b36659200/45929032 bytes (79.8%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b40689664/45929032 bytes (88.6%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b44728320/45929032 bytes (97.4%)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b45929032/45929032 bytes (100.0%)\n", " Done\n", "File saved as /root/.imageio/ffmpeg/ffmpeg-linux64-v3.3.1.\n", "TensorFlow version 1.15.3\n", "Sonnet version 1.36\n" ] } ], "source": [ "import tensorflow as tf\n", "import sonnet \n", "from moviepy.editor import *\n", "from IPython.display import Image\n", "import numpy as np\n", "import cv2\n", "\n", "print(\"TensorFlow version {}\".format(tf.__version__))\n", "print(\"Sonnet version {}\".format(sonnet.__version__))" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 104 }, "colab_type": "code", "id": "7qq4CI1m7gIz", "outputId": "00956fa4-b950-44b2-8d25-5be48e8eafce" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Cloning into 'kinetics-i3d'...\n", "remote: Enumerating objects: 69, done.\u001b[K\n", "remote: Total 69 (delta 0), reused 0 (delta 0), pack-reused 69\u001b[K\n", "Unpacking objects: 100% (69/69), done.\n", "Checking out files: 100% (40/40), done.\n" ] } ], "source": [ "!git clone https://github.com/deepmind/kinetics-i3d" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "EVfzKFqm_u6C" }, "source": [ "## Get data" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 208 }, "colab_type": "code", "id": "9MmJzD56BBUF", "outputId": "2e0689ab-bf4c-4170-92f3-2f674496ea27" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--2020-06-20 18:04:47-- https://storage.googleapis.com/cs231n-emotiw/data/val-tiny.zip\n", "Resolving storage.googleapis.com (storage.googleapis.com)... 64.233.189.128, 2404:6800:4008:c05::80\n", "Connecting to storage.googleapis.com (storage.googleapis.com)|64.233.189.128|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 48020782 (46M) [application/zip]\n", "Saving to: ‘val-tiny.zip’\n", "\n", "val-tiny.zip 100%[===================>] 45.80M 54.7MB/s in 0.8s \n", "\n", "2020-06-20 18:04:49 (54.7 MB/s) - ‘val-tiny.zip’ saved [48020782/48020782]\n", "\n" ] } ], "source": [ "!wget https://storage.googleapis.com/cs231n-emotiw/data/val-tiny.zip" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 901 }, "colab_type": "code", "id": "SKbMlzowBWR1", "outputId": "b95b98b2-4701-4019-b1a1-c65bbf608fff" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Archive: val-tiny.zip\n", " inflating: ./tiny/11_10.mp4 \n", " inflating: ./tiny/11_16.mp4 \n", " inflating: ./tiny/115_10.mp4 \n", " inflating: ./tiny/115_8.mp4 \n", " inflating: ./tiny/115_9.mp4 \n", " inflating: ./tiny/131_6.mp4 \n", " inflating: ./tiny/145_4.mp4 \n", " inflating: ./tiny/157_10.mp4 \n", " inflating: ./tiny/179_5.mp4 \n", " inflating: ./tiny/181_2.mp4 \n", " inflating: ./tiny/181_5.mp4 \n", " inflating: ./tiny/181_8.mp4 \n", " inflating: ./tiny/268_1.mp4 \n", " inflating: ./tiny/27_13.mp4 \n", " inflating: ./tiny/27_17.mp4 \n", " inflating: ./tiny/274_2.mp4 \n", " inflating: ./tiny/284_11.mp4 \n", " inflating: ./tiny/284_9.mp4 \n", " inflating: ./tiny/289_17.mp4 \n", " inflating: ./tiny/289_2.mp4 \n", " inflating: ./tiny/289_34.mp4 \n", " inflating: ./tiny/289_42.mp4 \n", " inflating: ./tiny/294_20.mp4 \n", " inflating: ./tiny/294_26.mp4 \n", " inflating: ./tiny/294_30.mp4 \n", " inflating: ./tiny/294_39.mp4 \n", " inflating: ./tiny/294_9.mp4 \n", " inflating: ./tiny/298_21.mp4 \n", " inflating: ./tiny/298_29.mp4 \n", " inflating: ./tiny/298_31.mp4 \n", " inflating: ./tiny/298_9.mp4 \n", " inflating: ./tiny/306_8.mp4 \n", " inflating: ./tiny/320_1.mp4 \n", " inflating: ./tiny/320_2.mp4 \n", " inflating: ./tiny/320_8.mp4 \n", " inflating: ./tiny/325_37.mp4 \n", " inflating: ./tiny/44_14.mp4 \n", " inflating: ./tiny/44_2.mp4 \n", " inflating: ./tiny/5_17.mp4 \n", " inflating: ./tiny/53_12.mp4 \n", " inflating: ./tiny/53_31.mp4 \n", " inflating: ./tiny/53_7.mp4 \n", " inflating: ./tiny/70_3.mp4 \n", " inflating: ./tiny/77_4.mp4 \n", " inflating: ./tiny/81_21.mp4 \n", " inflating: ./tiny/86_16.mp4 \n", " inflating: ./tiny/86_2.mp4 \n", " inflating: ./tiny/86_39.mp4 \n", " inflating: ./tiny/86_48.mp4 \n", " inflating: ./tiny/86_4.mp4 \n" ] } ], "source": [ "!mkdir tiny/\n", "!unzip val-tiny.zip -d ./tiny/" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 156 }, "colab_type": "code", "id": "bDADfE5mB8H1", "outputId": "7bcf399e-432c-47ee-a3aa-82b8002efb09" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "11_10.mp4 179_5.mp4 284_11.mp4 294_30.mp4 320_1.mp4 53_31.mp4 86_48.mp4\n", "11_16.mp4 181_2.mp4 284_9.mp4 294_39.mp4 320_2.mp4 53_7.mp4 86_4.mp4\n", "115_10.mp4 181_5.mp4 289_17.mp4 294_9.mp4 320_8.mp4 70_3.mp4\n", "115_8.mp4 181_8.mp4 289_2.mp4 298_21.mp4 325_37.mp4 77_4.mp4\n", "115_9.mp4 268_1.mp4 289_34.mp4 298_29.mp4 44_14.mp4 81_21.mp4\n", "131_6.mp4 27_13.mp4 289_42.mp4 298_31.mp4 44_2.mp4 86_16.mp4\n", "145_4.mp4 27_17.mp4 294_20.mp4 298_9.mp4 5_17.mp4 86_2.mp4\n", "157_10.mp4 274_2.mp4 294_26.mp4 306_8.mp4 53_12.mp4 86_39.mp4\n" ] } ], "source": [ "!cd tiny ; ls" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "colab": {}, "colab_type": "code", "id": "pYNxGvLARCgk" }, "outputs": [], "source": [ "!mv 'kinetics-i3d' kineticsi3d " ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "Ih_XkboknPOd" }, "source": [ "### The i3d Model" ] }, { "cell_type": "code", "execution_count": 75, "metadata": { "colab": {}, "colab_type": "code", "id": "k5o_hnqES-1-" }, "outputs": [], "source": [ "# Adapted from the kinetics-i3d repository from Google's DeepMind\n", "# https://github.com/deepmind/kinetics-i3d \n", "\n", "import numpy as np\n", "import tensorflow as tf\n", "\n", "from kineticsi3d import i3d\n", "\n", "_IMAGE_SIZE = 224\n", "\n", "\n", "_CHECKPOINT_PATHS = {\n", " 'rgb': './kineticsi3d/data/checkpoints/rgb_scratch/model.ckpt',\n", " 'rgb600': './kineticsi3d/data/checkpoints/rgb_scratch_kin600/model.ckpt',\n", " 'flow': './kineticsi3d/data/checkpoints/flow_scratch/model.ckpt',\n", " 'rgb_imagenet': './kineticsi3d/data/checkpoints/rgb_imagenet/model.ckpt',\n", " 'flow_imagenet': './kineticsi3d/data/checkpoints/flow_imagenet/model.ckpt',\n", "}\n", "\n", "_LABEL_MAP_PATH = './kineticsi3d/data/label_map.txt'\n", "_LABEL_MAP_PATH_600 = './kineticsi3d/data/label_map_600.txt'\n", "\n", "# FLAGS = tf.flags.FLAGS\n", "\n", "# tf.flags.DEFINE_string('eval_type', 'joint', 'rgb, rgb600, flow, or joint')\n", "# tf.flags.DEFINE_boolean('imagenet_pretrained', True, '')\n", "\n", "\n", "def get_actions(eval_type , video_arr , imagenet_pretrained=True , video_frames=151):\n", " tf.logging.set_verbosity(tf.logging.INFO)\n", " tf.reset_default_graph() \n", "\n", " NUM_CLASSES = 400\n", " if eval_type == 'rgb600':\n", " NUM_CLASSES = 600\n", "\n", " if eval_type not in ['rgb', 'rgb600', 'flow', 'joint']:\n", " raise ValueError('Bad `eval_type`, must be one of rgb, rgb600, flow, joint')\n", "\n", " if eval_type == 'rgb600':\n", " kinetics_classes = [x.strip() for x in open(_LABEL_MAP_PATH_600)]\n", " else:\n", " kinetics_classes = [x.strip() for x in open(_LABEL_MAP_PATH)]\n", "\n", " if eval_type in ['rgb', 'rgb600', 'joint']:\n", " # RGB input has 3 channels.\n", " rgb_input = tf.placeholder(\n", " tf.float32,\n", " shape=(1, video_frames, _IMAGE_SIZE, _IMAGE_SIZE, 3))\n", "\n", "\n", " with tf.variable_scope('RGB'):\n", " rgb_model = i3d.InceptionI3d(\n", " NUM_CLASSES, spatial_squeeze=True, final_endpoint='Logits')\n", " rgb_logits, _ = rgb_model(\n", " rgb_input, is_training=False, dropout_keep_prob=1.0)\n", "\n", "\n", " rgb_variable_map = {}\n", " for variable in tf.global_variables():\n", "\n", " if variable.name.split('/')[0] == 'RGB':\n", " if eval_type == 'rgb600':\n", " rgb_variable_map[variable.name.replace(':0', '')[len('RGB/inception_i3d/'):]] = variable\n", " else:\n", " rgb_variable_map[variable.name.replace(':0', '')] = variable\n", "\n", " rgb_saver = tf.train.Saver(var_list=rgb_variable_map, reshape=True)\n", "\n", " if eval_type in ['flow', 'joint']:\n", " # Flow input has only 2 channels.\n", " flow_input = tf.placeholder(\n", " tf.float32,\n", " shape=(1, video_frames, _IMAGE_SIZE, _IMAGE_SIZE, 2))\n", " with tf.variable_scope('Flow'):\n", " flow_model = i3d.InceptionI3d(\n", " NUM_CLASSES, spatial_squeeze=True, final_endpoint='Logits')\n", " flow_logits, _ = flow_model(\n", " flow_input, is_training=False, dropout_keep_prob=1.0)\n", " flow_variable_map = {}\n", " for variable in tf.global_variables():\n", " if variable.name.split('/')[0] == 'Flow':\n", " flow_variable_map[variable.name.replace(':0', '')] = variable\n", " flow_saver = tf.train.Saver(var_list=flow_variable_map, reshape=True)\n", "\n", " if eval_type == 'rgb' or eval_type == 'rgb600':\n", " model_logits = rgb_logits\n", " elif eval_type == 'flow':\n", " model_logits = flow_logits\n", " else:\n", " model_logits = rgb_logits + flow_logits\n", " model_predictions = tf.nn.softmax(model_logits)\n", "\n", " with tf.Session() as sess:\n", " feed_dict = {}\n", " if eval_type in ['rgb', 'rgb600', 'joint']:\n", " if imagenet_pretrained:\n", " rgb_saver.restore(sess, _CHECKPOINT_PATHS['rgb_imagenet'])\n", " else:\n", " rgb_saver.restore(sess, _CHECKPOINT_PATHS[eval_type])\n", " tf.logging.info('RGB checkpoint restored')\n", " rgb_sample = video_arr\n", " tf.logging.info('RGB data loaded, shape=%s', str(rgb_sample.shape))\n", " feed_dict[rgb_input] = rgb_sample\n", "\n", " if eval_type in ['flow', 'joint']:\n", " if imagenet_pretrained:\n", " flow_saver.restore(sess, _CHECKPOINT_PATHS['flow_imagenet'])\n", " else:\n", " flow_saver.restore(sess, _CHECKPOINT_PATHS['flow'])\n", " tf.logging.info('Flow checkpoint restored')\n", " flow_sample = video_arr\n", " tf.logging.info('Flow data loaded, shape=%s', str(flow_sample.shape))\n", " feed_dict[flow_input] = flow_sample\n", "\n", " out_logits, out_predictions = sess.run(\n", " [model_logits, model_predictions],\n", " feed_dict=feed_dict)\n", "\n", " out_logits = out_logits[0]\n", " out_predictions = out_predictions[0]\n", " sorted_indices = np.argsort(out_predictions)[::-1]\n", "\n", " # print('Norm of logits: %f' % np.linalg.norm(out_logits))\n", " # print('\\nTop classes and probabilities')\n", " out_pred = []\n", " out_kinetics_classes = []\n", " out_logits_final = []\n", " for index in sorted_indices[:20]:\n", "\n", " #print(out_predictions[index], out_logits[index], kinetics_classes[index])\n", " out_pred.append(out_predictions[index])\n", " out_kinetics_classes.append(kinetics_classes[index])\n", " out_logits_final.append(out_logits[index])\n", "\n", " return out_pred , out_logits_final , out_kinetics_classes\n" ] }, { "cell_type": "code", "execution_count": 77, "metadata": { "colab": {}, "colab_type": "code", "id": "FUSylmOhJv2A" }, "outputs": [], "source": [ "def normalize_video_arr(input_arr):\n", " input_arr_copy = 2 * ( (input_arr - np.min(input_arr)) / (np.max(input_arr) - np.min(input_arr))) - 1\n", " #input_arr_copy = np.expand_dims(input_arr_copy , 0)\n", " #(input_arr_copy.shape)\n", " return input_arr_copy" ] }, { "cell_type": "code", "execution_count": 74, "metadata": { "colab": {}, "colab_type": "code", "id": "YZPyVRqab589" }, "outputs": [], "source": [ "def preprocess_action(video_path):\n", " '''\n", " Input: the video path \n", "\n", " Output\n", " out_predictions = The confidence score of the model for the prediction\n", " out_logits = The output logits\n", " kinetics_classes = The string array of the top 20 ranked kinetic predictions.\n", "\n", " '''\n", " clip_resize = VideoFileClip(video_path).resize(height=256)\n", " clip_resize.write_videofile(video_path[:-4] + '-resize.mp4')\n", " cap = cv2.VideoCapture(video_path[:-4] + '-resize.mp4')\n", " frameCount = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n", " frameWidth = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n", " frameHeight = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n", "\n", " buf = np.empty((1, frameCount, 224, 224, 3), np.dtype('uint8'))\n", " \n", " fc = 0\n", " ret = True\n", "\n", " i = 0\n", " while (fc < frameCount and ret):\n", " ret, tmp = cap.read()\n", " # if i % 10 == 0:\n", " buf[0][fc] = cv2.resize(tmp, (224 , 224))\n", " fc += 1\n", " # if fc >= 10:\n", " \n", " i += 1\n", "\n", " cap.release()\n", "\n", " np_vid = np.asarray(buf)\n", "\n", " np_vid = normalize_video_arr(np_vid)\n", " return get_actions(eval_type='rgb' , video_arr=np_vid, video_frames=np_vid.shape[1])\n", "\n", " \n" ] }, { "cell_type": "code", "execution_count": 78, "metadata": { "colab": {}, "colab_type": "code", "id": "YwpokPA6lRSY" }, "outputs": [], "source": [ "def display_video(video_path):\n", " clip = (VideoFileClip(video_path)\n", " .subclip((0,0),(0,5.0))\n", " .resize(height=256))\n", "\n", " clip.write_gif(video_path[:-4] + \".gif\" , verbose= False , progress_bar = False)\n", "\n", " with open(video_path[:-4] + \".gif\",'rb') as f:\n", " display(Image(data=f.read(), format='png'))" ] }, { "cell_type": "code", "execution_count": 79, "metadata": { "colab": {}, "colab_type": "code", "id": "uLxAqwYfUl39" }, "outputs": [], "source": [ "def display_action(video_path):\n", "\n", " _, _, kinetics_classes = preprocess_action(video_path)\n", "\n", " print('Predicted Action: ' , kinetics_classes[0])\n", "\n", " display_video(video_path=video_path)\n", "\n", "\n" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "ssPU7eDrNZdG" }, "source": [ "### Dry Run" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1000 }, "colab_type": "code", "id": "AJSBDsgueIqw", "outputId": "bd112f9e-b53d-4127-ec17-d79401b8008b" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[MoviePy] >>>> Building video /content/tiny/274_2-resize.mp4\n", "[MoviePy] Writing audio in 274_2-resizeTEMP_MPY_wvf_snd.mp3\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 111/111 [00:00<00:00, 1050.82it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[MoviePy] Done.\n", "[MoviePy] Writing video /content/tiny/274_2-resize.mp4\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n", "100%|██████████| 126/126 [00:01<00:00, 75.89it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[MoviePy] Done.\n", "[MoviePy] >>>> Video ready: /content/tiny/274_2-resize.mp4 \n", "\n", "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/sonnet/python/modules/conv.py:325: calling TruncatedNormal.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Call initializer instance with the dtype argument instead of passing it to the constructor\n", "WARNING:tensorflow:From /content/kineticsi3d/i3d.py:463: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n", "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/sonnet/python/modules/conv.py:330: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Call initializer instance with the dtype argument instead of passing it to the constructor\n", "INFO:tensorflow:Restoring parameters from ./kineticsi3d/data/checkpoints/rgb_imagenet/model.ckpt\n", "INFO:tensorflow:RGB checkpoint restored\n", "INFO:tensorflow:RGB data loaded, shape=(1, 126, 224, 224, 3)\n", "marching\n", "[MoviePy] >>>> Building video /content/tiny/27_17-resize.mp4\n", "[MoviePy] Writing audio in 27_17-resizeTEMP_MPY_wvf_snd.mp3\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 111/111 [00:00<00:00, 1265.24it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[MoviePy] Done.\n", "[MoviePy] Writing video /content/tiny/27_17-resize.mp4\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n", "100%|██████████| 66/66 [00:00<00:00, 231.89it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[MoviePy] Done.\n", "[MoviePy] >>>> Video ready: /content/tiny/27_17-resize.mp4 \n", "\n", "INFO:tensorflow:Restoring parameters from ./kineticsi3d/data/checkpoints/rgb_imagenet/model.ckpt\n", "INFO:tensorflow:RGB checkpoint restored\n", "INFO:tensorflow:RGB data loaded, shape=(1, 66, 224, 224, 3)\n", "singing\n", "[MoviePy] >>>> Building video /content/tiny/298_31-resize.mp4\n", "[MoviePy] Writing audio in 298_31-resizeTEMP_MPY_wvf_snd.mp3\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 111/111 [00:00<00:00, 995.95it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[MoviePy] Done.\n", "[MoviePy] Writing video /content/tiny/298_31-resize.mp4\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n", "100%|██████████| 126/126 [00:01<00:00, 93.67it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[MoviePy] Done.\n", "[MoviePy] >>>> Video ready: /content/tiny/298_31-resize.mp4 \n", "\n", "INFO:tensorflow:Restoring parameters from ./kineticsi3d/data/checkpoints/rgb_imagenet/model.ckpt\n", "INFO:tensorflow:RGB checkpoint restored\n", "INFO:tensorflow:RGB data loaded, shape=(1, 126, 224, 224, 3)\n", "testifying\n", "[MoviePy] >>>> Building video /content/tiny/11_16-resize.mp4\n", "[MoviePy] Writing audio in 11_16-resizeTEMP_MPY_wvf_snd.mp3\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 111/111 [00:00<00:00, 1206.65it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[MoviePy] Done.\n", "[MoviePy] Writing video /content/tiny/11_16-resize.mp4\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n", "100%|██████████| 151/151 [00:01<00:00, 85.75it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[MoviePy] Done.\n", "[MoviePy] >>>> Video ready: /content/tiny/11_16-resize.mp4 \n", "\n", "INFO:tensorflow:Restoring parameters from ./kineticsi3d/data/checkpoints/rgb_imagenet/model.ckpt\n", "INFO:tensorflow:RGB checkpoint restored\n", "INFO:tensorflow:RGB data loaded, shape=(1, 151, 224, 224, 3)\n", "news anchoring\n", "[MoviePy] >>>> Building video /content/tiny/294_26-resize.mp4\n", "[MoviePy] Writing audio in 294_26-resizeTEMP_MPY_wvf_snd.mp3\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 111/111 [00:00<00:00, 1003.55it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[MoviePy] Done.\n", "[MoviePy] Writing video /content/tiny/294_26-resize.mp4\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n", "100%|██████████| 126/126 [00:01<00:00, 94.85it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[MoviePy] Done.\n", "[MoviePy] >>>> Video ready: /content/tiny/294_26-resize.mp4 \n", "\n", "INFO:tensorflow:Restoring parameters from ./kineticsi3d/data/checkpoints/rgb_imagenet/model.ckpt\n", "INFO:tensorflow:RGB checkpoint restored\n", "INFO:tensorflow:RGB data loaded, shape=(1, 126, 224, 224, 3)\n", "stretching arm\n", "[MoviePy] >>>> Building video /content/tiny/115_8-resize.mp4\n", "[MoviePy] Writing audio in 115_8-resizeTEMP_MPY_wvf_snd.mp3\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 111/111 [00:00<00:00, 1038.28it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[MoviePy] Done.\n", "[MoviePy] Writing video /content/tiny/115_8-resize.mp4\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n", "100%|██████████| 126/126 [00:00<00:00, 186.64it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[MoviePy] Done.\n", "[MoviePy] >>>> Video ready: /content/tiny/115_8-resize.mp4 \n", "\n", "INFO:tensorflow:Restoring parameters from ./kineticsi3d/data/checkpoints/rgb_imagenet/model.ckpt\n", "INFO:tensorflow:RGB checkpoint restored\n", "INFO:tensorflow:RGB data loaded, shape=(1, 126, 224, 224, 3)\n", "celebrating\n", "[MoviePy] >>>> Building video /content/tiny/298_9-resize.mp4\n", "[MoviePy] Writing audio in 298_9-resizeTEMP_MPY_wvf_snd.mp3\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 111/111 [00:00<00:00, 1063.12it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[MoviePy] Done.\n", "[MoviePy] Writing video /content/tiny/298_9-resize.mp4\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n", "100%|██████████| 126/126 [00:01<00:00, 91.47it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[MoviePy] Done.\n", "[MoviePy] >>>> Video ready: /content/tiny/298_9-resize.mp4 \n", "\n", "INFO:tensorflow:Restoring parameters from ./kineticsi3d/data/checkpoints/rgb_imagenet/model.ckpt\n", "INFO:tensorflow:RGB checkpoint restored\n", "INFO:tensorflow:RGB data loaded, shape=(1, 126, 224, 224, 3)\n", "yoga\n", "[MoviePy] >>>> Building video /content/tiny/284_11-resize.mp4\n", "[MoviePy] Writing audio in 284_11-resizeTEMP_MPY_wvf_snd.mp3\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 111/111 [00:00<00:00, 823.73it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[MoviePy] Done.\n", "[MoviePy] Writing video /content/tiny/284_11-resize.mp4\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n", "100%|██████████| 151/151 [00:01<00:00, 87.22it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[MoviePy] Done.\n", "[MoviePy] >>>> Video ready: /content/tiny/284_11-resize.mp4 \n", "\n", "INFO:tensorflow:Restoring parameters from ./kineticsi3d/data/checkpoints/rgb_imagenet/model.ckpt\n", "INFO:tensorflow:RGB checkpoint restored\n", "INFO:tensorflow:RGB data loaded, shape=(1, 151, 224, 224, 3)\n", "gargling\n", "[MoviePy] >>>> Building video /content/tiny/145_4-resize.mp4\n", "[MoviePy] Writing audio in 145_4-resizeTEMP_MPY_wvf_snd.mp3\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 111/111 [00:00<00:00, 864.05it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[MoviePy] Done.\n", "[MoviePy] Writing video /content/tiny/145_4-resize.mp4\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n", "100%|██████████| 151/151 [00:02<00:00, 74.41it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[MoviePy] Done.\n", "[MoviePy] >>>> Video ready: /content/tiny/145_4-resize.mp4 \n", "\n", "INFO:tensorflow:Restoring parameters from ./kineticsi3d/data/checkpoints/rgb_imagenet/model.ckpt\n", "INFO:tensorflow:RGB checkpoint restored\n", "INFO:tensorflow:RGB data loaded, shape=(1, 151, 224, 224, 3)\n", "marching\n", "[MoviePy] >>>> Building video /content/tiny/77_4-resize.mp4\n", "[MoviePy] Writing audio in 77_4-resizeTEMP_MPY_wvf_snd.mp3\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 111/111 [00:00<00:00, 833.86it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[MoviePy] Done.\n", "[MoviePy] Writing video /content/tiny/77_4-resize.mp4\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n", "100%|██████████| 121/121 [00:01<00:00, 87.20it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "[MoviePy] Done.\n", "[MoviePy] >>>> Video ready: /content/tiny/77_4-resize.mp4 \n", "\n", "INFO:tensorflow:Restoring parameters from ./kineticsi3d/data/checkpoints/rgb_imagenet/model.ckpt\n", "INFO:tensorflow:RGB checkpoint restored\n", "INFO:tensorflow:RGB data loaded, shape=(1, 121, 224, 224, 3)\n", "canoeing or kayaking\n" ] } ], "source": [ "import glob\n", "counter = 0\n", "LIMIT = 10\n", "\n", "videos = glob.glob('/content/tiny/*.mp4')\n", "\n", "stored_kinetic_labels = {}\n", "\n", "for vid in videos:\n", " if 'resize' not in vid:\n", " _, _, kinetics_classes = preprocess_action(vid)\n", " print(kinetics_classes[0])\n", "\n", " stored_kinetic_labels[vid] = kinetics_classes[0]\n", "\n", " counter += 1\n", "\n", " if counter >= LIMIT:\n", " break" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 55 }, "colab_type": "code", "id": "WUyOl-CNOgI0", "outputId": "e24f92c4-cbda-48d7-e18c-53f608166730" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'/content/tiny/274_2.mp4': 'marching', '/content/tiny/27_17.mp4': 'singing', '/content/tiny/298_31.mp4': 'testifying', '/content/tiny/11_16.mp4': 'news anchoring', '/content/tiny/294_26.mp4': 'stretching arm', '/content/tiny/115_8.mp4': 'celebrating', '/content/tiny/298_9.mp4': 'yoga', '/content/tiny/284_11.mp4': 'gargling', '/content/tiny/145_4.mp4': 'marching', '/content/tiny/77_4.mp4': 'canoeing or kayaking'}\n" ] } ], "source": [ "print(stored_kinetic_labels)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1000 }, "colab_type": "code", "id": "i0LiVmyOkItz", "outputId": "c8842d36-9e75-4ed7-a466-4b8709b1b245" }, "outputs": [], "source": [ "display_action('/content/tiny/274_2.mp4')\n" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "hqDLIf_XVxkH" }, "source": [ "## Collect actions for Entire Dataset" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 391 }, "colab_type": "code", "id": "Kv7wOAfMWaDr", "outputId": "c96926b2-cebc-405d-8307-5bc6aab5d57a" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--2020-06-20 15:30:30-- https://storage.googleapis.com/cs231n-emotiw/data/train-full.zip\n", "Resolving storage.googleapis.com (storage.googleapis.com)... 74.125.195.128, 2607:f8b0:400e:c08::80\n", "Connecting to storage.googleapis.com (storage.googleapis.com)|74.125.195.128|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 2732428104 (2.5G) [application/zip]\n", "Saving to: ‘train-full.zip’\n", "\n", "train-full.zip 100%[===================>] 2.54G 73.3MB/s in 36s \n", "\n", "2020-06-20 15:31:06 (73.1 MB/s) - ‘train-full.zip’ saved [2732428104/2732428104]\n", "\n", "--2020-06-20 15:31:13-- https://storage.googleapis.com/cs231n-emotiw/data/val-full.zip\n", "Resolving storage.googleapis.com (storage.googleapis.com)... 74.125.135.128, 2607:f8b0:400e:c03::80\n", "Connecting to storage.googleapis.com (storage.googleapis.com)|74.125.135.128|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 740388884 (706M) [application/zip]\n", "Saving to: ‘val-full.zip’\n", "\n", "val-full.zip 100%[===================>] 706.09M 84.5MB/s in 9.3s \n", "\n", "2020-06-20 15:31:23 (75.6 MB/s) - ‘val-full.zip’ saved [740388884/740388884]\n", "\n" ] } ], "source": [ "!wget https://storage.googleapis.com/cs231n-emotiw/data/train-full.zip\n", "!wget https://storage.googleapis.com/cs231n-emotiw/data/val-full.zip" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 399 }, "colab_type": "code", "id": "SOnxAvhUXEP-", "outputId": "e96d06c9-2267-486f-923f-d94f713939e8" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--2020-06-20 18:07:06-- https://storage.googleapis.com/cs231n-emotiw/data/Train_labels.txt\n", "Resolving storage.googleapis.com (storage.googleapis.com)... 74.125.23.128, 2404:6800:4008:c04::80\n", "Connecting to storage.googleapis.com (storage.googleapis.com)|74.125.23.128|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 21653 (21K) [text/plain]\n", "Saving to: ‘Train_labels.txt’\n", "\n", "Train_labels.txt 100%[===================>] 21.15K --.-KB/s in 0s \n", "\n", "2020-06-20 18:07:07 (69.2 MB/s) - ‘Train_labels.txt’ saved [21653/21653]\n", "\n", "--2020-06-20 18:07:07-- https://storage.googleapis.com/cs231n-emotiw/data/Val_labels.txt\n", "Resolving storage.googleapis.com (storage.googleapis.com)... 64.233.189.128, 2404:6800:4008:c01::80\n", "Connecting to storage.googleapis.com (storage.googleapis.com)|64.233.189.128|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 6265 (6.1K) [text/plain]\n", "Saving to: ‘Val_labels.txt’\n", "\n", "Val_labels.txt 100%[===================>] 6.12K --.-KB/s in 0s \n", "\n", "2020-06-20 18:07:08 (71.5 MB/s) - ‘Val_labels.txt’ saved [6265/6265]\n", "\n" ] } ], "source": [ "!wget https://storage.googleapis.com/cs231n-emotiw/data/Train_labels.txt\n", "!wget https://storage.googleapis.com/cs231n-emotiw/data/Val_labels.txt" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "TZIcIgqu6EQj" }, "source": [ "## Action Preprocessor Class" ] }, { "cell_type": "code", "execution_count": 60, "metadata": { "colab": {}, "colab_type": "code", "id": "UkfCvxLGBU3x" }, "outputs": [], "source": [ "import zipfile\n", "from concurrent.futures import ThreadPoolExecutor\n", "from pathlib import Path\n", "import os\n", "import tensorflow as tf\n", "from os.path import isfile, join\n", "import pickle\n", "import glob\n", "import subprocess\n", "from tqdm import tqdm\n", "import numpy as np\n", "import tensorflow as tf\n", "import sonnet \n", "from moviepy.editor import *\n", "from IPython.display import Image\n", "import cv2\n", "from sklearn.preprocessing import OneHotEncoder\n", "from sklearn.preprocessing import LabelEncoder\n", "\n", "class ActionPreprocessor:\n", " \"\"\"\n", " Extract the actions from the videos.\n", " Faces are stored in a flat directory structure (no categorical hierarchy)\n", " \n", "\n", " \"\"\"\n", "\n", " def __init__(self, video_folder, output_folder, output_file=None, label_path = None , is_zip=True):\n", " \"\"\"\n", " @param video_folder The folder where the list of videos frames are stored. If \n", " `is_zip` is set to True, this should be a single zip \n", " file containing the video frames. Paths can either by a local \n", " folder or a GDrive mounted path.\n", " @param output_folder The local output path where the preprocessed files will be stored for \n", " further preprocessing can be done\n", " @param output_file If not none, the output_folder will be zipped up and stored at this location\n", " @param label_path The path of the .txt file containing the class labels matched to the sample name.\n", " @param is_zip If set to True, the `video_folder` will be unzipped prior to accessing\n", " - hop_size: The frame collection rate \n", " @param sample_every the frames to skip.\n", "\n", " \"\"\"\n", " self.is_zip = is_zip\n", " self.video_folder = video_folder\n", " self.output_folder = output_folder\n", " self.output_file = output_file\n", " self.label_path = label_path\n", "\n", " \n", " print(f\"Action Preprocessor created with is_zip = {is_zip}, video_folder = {video_folder} , output_folder = {output_folder}, output_file = {output_file}\")\n", " \n", " def display_video(self, video_path):\n", " clip = (VideoFileClip(video_path)\n", " .subclip((0,0),(0,5.0))\n", " .resize(height=256))\n", "\n", " clip.write_gif(video_path[:-4] + \".gif\")\n", "\n", "\n", " with open(video_path[:-4] + \".gif\",'rb') as f:\n", " display(Image(data=f.read(), format='png'))\n", "\n", " def normalize_video_arr(self, input_arr):\n", " input_arr_copy = 2 * ( (input_arr - np.min(input_arr)) / (np.max(input_arr) - np.min(input_arr))) - 1\n", " return input_arr_copy \n", "\n", " def get_actions(self, eval_type , video_arr , imagenet_pretrained=True , video_frames=151):\n", "\n", " from kineticsi3d import i3d\n", "\n", " _IMAGE_SIZE = 224\n", "\n", "\n", " _CHECKPOINT_PATHS = {\n", " 'rgb': './kineticsi3d/data/checkpoints/rgb_scratch/model.ckpt',\n", " 'rgb600': './kineticsi3d/data/checkpoints/rgb_scratch_kin600/model.ckpt',\n", " 'flow': './kineticsi3d/data/checkpoints/flow_scratch/model.ckpt',\n", " 'rgb_imagenet': './kineticsi3d/data/checkpoints/rgb_imagenet/model.ckpt',\n", " 'flow_imagenet': './kineticsi3d/data/checkpoints/flow_imagenet/model.ckpt',\n", " }\n", "\n", " _LABEL_MAP_PATH = './kineticsi3d/data/label_map.txt'\n", " _LABEL_MAP_PATH_600 = './kineticsi3d/data/label_map_600.txt'\n", "\n", " tf.logging.set_verbosity(tf.logging.ERROR)\n", " tf.reset_default_graph() \n", "\n", " NUM_CLASSES = 400\n", " if eval_type == 'rgb600':\n", " NUM_CLASSES = 600\n", "\n", " if eval_type not in ['rgb', 'rgb600', 'flow', 'joint']:\n", " raise ValueError('Bad `eval_type`, must be one of rgb, rgb600, flow, joint')\n", "\n", " if eval_type == 'rgb600':\n", " kinetics_classes = [x.strip() for x in open(_LABEL_MAP_PATH_600)]\n", " else:\n", " kinetics_classes = [x.strip() for x in open(_LABEL_MAP_PATH)]\n", "\n", " if eval_type in ['rgb', 'rgb600', 'joint']:\n", " # RGB input has 3 channels.\n", " rgb_input = tf.placeholder(\n", " tf.float32,\n", " shape=(1, video_frames, _IMAGE_SIZE, _IMAGE_SIZE, 3))\n", "\n", "\n", " with tf.variable_scope('RGB'):\n", " rgb_model = i3d.InceptionI3d(\n", " NUM_CLASSES, spatial_squeeze=True, final_endpoint='Logits')\n", " rgb_logits, _ = rgb_model(\n", " rgb_input, is_training=False, dropout_keep_prob=1.0)\n", "\n", "\n", " rgb_variable_map = {}\n", " for variable in tf.global_variables():\n", "\n", " if variable.name.split('/')[0] == 'RGB':\n", " if eval_type == 'rgb600':\n", " rgb_variable_map[variable.name.replace(':0', '')[len('RGB/inception_i3d/'):]] = variable\n", " else:\n", " rgb_variable_map[variable.name.replace(':0', '')] = variable\n", "\n", " rgb_saver = tf.train.Saver(var_list=rgb_variable_map, reshape=True)\n", "\n", " if eval_type in ['flow', 'joint']:\n", " # Flow input has only 2 channels.\n", " flow_input = tf.placeholder(\n", " tf.float32,\n", " shape=(1, video_frames, _IMAGE_SIZE, _IMAGE_SIZE, 2))\n", " with tf.variable_scope('Flow'):\n", " flow_model = i3d.InceptionI3d(\n", " NUM_CLASSES, spatial_squeeze=True, final_endpoint='Logits')\n", " flow_logits, _ = flow_model(\n", " flow_input, is_training=False, dropout_keep_prob=1.0)\n", " flow_variable_map = {}\n", " for variable in tf.global_variables():\n", " if variable.name.split('/')[0] == 'Flow':\n", " flow_variable_map[variable.name.replace(':0', '')] = variable\n", " flow_saver = tf.train.Saver(var_list=flow_variable_map, reshape=True)\n", "\n", " if eval_type == 'rgb' or eval_type == 'rgb600':\n", " model_logits = rgb_logits\n", " elif eval_type == 'flow':\n", " model_logits = flow_logits\n", " else:\n", " model_logits = rgb_logits + flow_logits\n", " model_predictions = tf.nn.softmax(model_logits)\n", "\n", " with tf.Session() as sess:\n", " feed_dict = {}\n", " if eval_type in ['rgb', 'rgb600', 'joint']:\n", " if imagenet_pretrained:\n", " rgb_saver.restore(sess, _CHECKPOINT_PATHS['rgb_imagenet'])\n", " else:\n", " rgb_saver.restore(sess, _CHECKPOINT_PATHS[eval_type])\n", " tf.logging.info('RGB checkpoint restored')\n", " rgb_sample = video_arr\n", " tf.logging.info('RGB data loaded, shape=%s', str(rgb_sample.shape))\n", " feed_dict[rgb_input] = rgb_sample\n", "\n", " if eval_type in ['flow', 'joint']:\n", " if imagenet_pretrained:\n", " flow_saver.restore(sess, _CHECKPOINT_PATHS['flow_imagenet'])\n", " else:\n", " flow_saver.restore(sess, _CHECKPOINT_PATHS['flow'])\n", " tf.logging.info('Flow checkpoint restored')\n", " flow_sample = video_arr\n", " tf.logging.info('Flow data loaded, shape=%s', str(flow_sample.shape))\n", " feed_dict[flow_input] = flow_sample\n", "\n", " out_logits, out_predictions = sess.run(\n", " [model_logits, model_predictions],\n", " feed_dict=feed_dict)\n", "\n", " out_logits = out_logits[0]\n", " out_predictions = out_predictions[0]\n", " sorted_indices = np.argsort(out_predictions)[::-1]\n", "\n", " # print('Norm of logits: %f' % np.linalg.norm(out_logits))\n", " # print('\\nTop classes and probabilities')\n", " out_pred = []\n", " out_kinetics_classes = []\n", " out_logits_final = []\n", " for index in sorted_indices[:20]:\n", "\n", " #print(out_predictions[index], out_logits[index], kinetics_classes[index])\n", " out_pred.append(out_predictions[index])\n", " out_kinetics_classes.append(kinetics_classes[index])\n", " out_logits_final.append(out_logits[index])\n", "\n", " return out_pred , out_logits_final , out_kinetics_classes\n", "\n", " def preprocess_action(self, video_path):\n", " '''\n", " Input: the video path \n", "\n", " Output\n", " out_predictions = The confidence score of the model for the prediction\n", " out_logits = The output logits\n", " kinetics_classes = The string array of the top 20 ranked kinetic predictions.\n", "\n", " '''\n", " #clip_resize = VideoFileClip(video_path).resize(height=256)\n", " #clip_resize.write_videofile(video_path[:-4] + '-resize.mp4' , verbose=False , progress_bar=False)\n", " cap = cv2.VideoCapture(video_path)\n", " frameCount = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n", " frameWidth = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n", " frameHeight = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n", "\n", " buf = np.empty((1, frameCount, 224, 224, 3), np.dtype('uint8'))\n", " \n", " fc = 0\n", " ret = True\n", "\n", " i = 0\n", " while (fc < frameCount and ret):\n", " ret, tmp = cap.read()\n", " # if i % 10 == 0:\n", " buf[0][fc] = cv2.resize(tmp, (224 , 224))\n", " fc += 1\n", " # if fc >= 10:\n", " \n", " i += 1\n", "\n", " cap.release()\n", "\n", " np_vid = np.asarray(buf)\n", "\n", " np_vid = self.normalize_video_arr(np_vid)\n", " return self.get_actions(eval_type='rgb' , video_arr=np_vid, video_frames=np_vid.shape[1])\n", " def preprocess(self):\n", " \"\"\"\n", " Outputs: Writes to disk the openl3 embedding pickle object for each sample. \n", " Optionally, it will output the entire matched X and Y numpy pickle objects if label path is provided\n", " - \n", " \"\"\"\n", "\n", "\n", " tmp_output_folder = \"\"\n", " if self.is_zip:\n", " # Unzips files to a temp directory\n", " tmp_output_folder = self.output_folder.rstrip('/') + \"_tmp\"\n", " print(f\"Unzipping files to temp dir {tmp_output_folder}...\")\n", " Path(f\"{tmp_output_folder}\").mkdir(parents=True, exist_ok=True)\n", " with zipfile.ZipFile(self.video_folder, 'r') as zip_ref:\n", " zip_ref.extractall(tmp_output_folder)\n", " print(\"Finished unzipping files\")\n", " else:\n", " tmp_output_folder = self.video_folder\n", " print(\"Skipping unzipping files as input is a folder\")\n", "\n", " Path(f\"{self.output_folder}/action-pickle/\").mkdir(parents=True, exist_ok=True)\n", " \n", " \n", " video_files = sorted(glob.glob(tmp_output_folder + '/*.mp4'))\n", "\n", " # Match the targets to the video\n", "\n", " target_labels = []\n", "\n", " output_action_list = []\n", "\n", " if self.label_path is not None: \n", " targets = []\n", " target_labels = np.genfromtxt(self.label_path , delimiter = ' ' , dtype='str')\n", "\n", " for vid in tqdm(video_files): \n", " newname = os.path.basename(vid)\n", " #print(f\"Reading file {newname} ...\")\n", "\n", " _, _, kinetics_classes = self.preprocess_action(vid)\n", "\n", " output_action_list.append(kinetics_classes[0])\n", "\n", " if self.label_path is not None: \n", " target_index = np.where(target_labels[: , 0] == newname[:-4])[0]\n", " target_index = int(target_index)\n", " target = int(target_labels[: , 1][target_index]) - 1\n", " targets.append(target)\n", "\n", "\n", " with open(f\"{self.output_folder}/sparse_action_list.pkl\", \"wb\") as f_out:\n", " label_encoder = LabelEncoder()\n", " encoded_actions = label_encoder.fit_transform(output_action_list)\n", " pickle.dump(encoded_actions , f_out)\n", "\n", " with open(f\"{self.output_folder}/string_action_list.pkl\", \"wb\") as f_out:\n", " output_action_list = np.asarray(output_action_list , dtype='str')\n", " pickle.dump(output_action_list , f_out)\n", "\n", " with open(f\"{self.output_folder}/one_hot_encoded_action_list.pkl\", \"wb\") as f_out:\n", " one_hot_encoder = OneHotEncoder()\n", " encoded_actions = np.asarray(output_action_list)\n", " encoded_actions = encoded_actions.reshape(-1 , 1)\n", " encoded_actions = one_hot_encoder.fit_transform(encoded_actions)\n", " pickle.dump(encoded_actions , f_out)\n", "\n", "\n", " if self.label_path is not None:\n", "\n", " targets = np.asarray(targets)\n", " with open(f\"{self.output_folder}/action_video_targets.pkl\", \"wb\") as f_out:\n", " pickle.dump(targets , f_out)\n", "\n", " if self.output_file is not None:\n", " print(f\"Starting to zip files to {self.output_file}\")\n", " def zipdir(path, ziph):\n", " for root, dirs, files in os.walk(path):\n", " folder = root[len(path):]\n", " for file in files:\n", " ziph.write(join(root, file), join(folder, file))\n", "\n", " zipf = zipfile.ZipFile(self.output_file, 'w', zipfile.ZIP_DEFLATED)\n", " zipdir(self.output_folder, zipf)\n", " zipf.close()\n", " print(f\"Done zipping files to {self.output_file}\")\n", " \n", " print(\"Done!\")\n" ] }, { "cell_type": "code", "execution_count": 61, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 55 }, "colab_type": "code", "id": "68UCGuUYZ1US", "outputId": "7c8917b6-275c-47b1-8396-b81d7af43eb2" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Action Preprocessor created with is_zip = False, video_folder = /content/tiny , output_folder = action_out, output_file = None\n" ] } ], "source": [ "action_preprocessor = ActionPreprocessor('/content/tiny' , 'action_out' , label_path='Val_labels.txt' ,is_zip=False)" ] }, { "cell_type": "code", "execution_count": 62, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1000 }, "colab_type": "code", "id": "QqmdQLVsbgLs", "outputId": "c9295b48-795b-4c85-f859-9156a022db06" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "\n", "\n", "\n", " 0%| | 0/50 [00:00