diff --git a/Silas_Video_Scale_and_Frame_Interpolate_Real_ESRGAN_(anime)_and_RIFE.ipynb b/Silas_Video_Scale_and_Frame_Interpolate_Real_ESRGAN_(anime)_and_RIFE.ipynb new file mode 100644 index 0000000..54c669c --- /dev/null +++ b/Silas_Video_Scale_and_Frame_Interpolate_Real_ESRGAN_(anime)_and_RIFE.ipynb @@ -0,0 +1,828 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [], + "collapsed_sections": [ + "6BPxh_VmVVIu" + ], + "machine_shape": "hm", + "include_colab_link": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IRDbDYYMQt_Y" + }, + "source": [ + "# Video upscaling and frame interpolation tool\n", + "\n", + "There was nothing super reliable available, so I cobbled this together.\n", + "\n", + "Tested on colab+ / google drive. You'll need both of these for it to work, probably.\n", + "\n", + "From:\n", + "- Real-ESRGAN [GitHub repo](https://github.com/xinntao/Real-ESRGAN). \n", + "- RIFE [GitHub repo](https://github.com/megvii-research/ECCV2022-RIFE).\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6BPxh_VmVVIu" + }, + "source": [ + "# ESRGAN prep\n", + "Before start, make sure that you choose\n", + "* Runtime Type = Python 3\n", + "* Hardware Accelerator = GPU\n", + "\n", + "in the **Runtime** menu -> **Change runtime type**\n", + "\n", + "Then, we clone the repository, set up the envrironment." + ] + }, + { + "cell_type": "code", + "source": [ + "## See what GPU you got\n", + "!nvidia-smi" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "RQJgGjNXjolo", + "outputId": "2663278d-0283-4160-9759-178b7a1dbb18" + }, + "execution_count": 1, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Mon Oct 3 06:36:41 2022 \n", + "+-----------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 460.32.03 Driver Version: 460.32.03 CUDA Version: 11.2 |\n", + "|-------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|===============================+======================+======================|\n", + "| 0 Tesla P100-PCIE... Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 36C P0 26W / 250W | 0MiB / 16280MiB | 0% Default |\n", + "| | | N/A |\n", + "+-------------------------------+----------------------+----------------------+\n", + " \n", + "+-----------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=============================================================================|\n", + "| No running processes found |\n", + "+-----------------------------------------------------------------------------+\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "#Params\n", + "working_dir = '/content/gdrive/MyDrive/' #@param {type:\"string\"}\n", + "original_video = 'Gs6.mp4' #@param {type:\"string\"}\n", + "x2_video = 'Gs6_x2.mp4' #@param {type:\"string\"}\n", + "hifps_x2_video = 'Gs6_x2_hifps.mp4' #@param {type:\"string\"}\n", + "\n", + "#@markdown Interpolate steps: 1=2x, 2=4x, 3=8x, 4=16x (approx)\n", + "#@markdown
For example, 3 will convert a 5fps video to 40fps.\n", + "interpolate_steps=3#@param {type:\"slider\", min:1, max:5, step:1}\n", + "\n", + "\n", + "## video_path = 'upload/Gs6.mp4' #@param {type:\"string\"}\n", + "\n", + "video_path = working_dir + original_video\n", + "x2_video_path = working_dir + x2_video\n", + "hifps_x2_video_path = working_dir + hifps_x2_video\n", + "\n", + "\n", + "#Mount Google Drive\n", + "from google.colab import drive\n", + "drive.mount('/content/gdrive')" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "5E3mWzgcny0l", + "outputId": "ab3d07b9-d712-43aa-b8c8-db9d83472e57" + }, + "execution_count": 2, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Mounted at /content/gdrive\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "GnpnrLfMV2jU", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "79ef58e6-a2e0-44b8-8312-9f05c7cb3b79" + }, + "source": [ + "# Clone Real-ESRGAN, install dependencies\n", + "\n", + "# Clone to gdrive if not there already\n", + "import os\n", + "if os.path.isdir('/content/gdrive/MyDrive/Real-ESRGAN'):\n", + " %cd /content/gdrive/MyDrive/Real-ESRGAN\n", + "elif os.path.isdir('/content/gdrive/MyDrive/'):\n", + " %cd /content/gdrive/MyDrive\n", + " !git clone https://github.com/xinntao/Real-ESRGAN.git\n", + " %cd Real-ESRGAN\n", + "\n", + "# Set up the environment\n", + "print(\"Started Installing libraries for Real-ESRGAN upscaling.\")\n", + "!pip -q install basicsr\n", + "!pip -q install facexlib\n", + "!pip -q install gfpgan\n", + "!pip -q install ffmpeg-python\n", + "!pip -q install -r requirements.txt\n", + "print(\"Running setup.py - this will take a while...\")\n", + "!python setup.py develop &> /dev/null\n", + "print(\"Finished Installing libraries for Real-ESRGAN upscaling.\")" + ], + "execution_count": 19, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "/content/gdrive/MyDrive/Real-ESRGAN\n", + "Started Installing libraries for Real-ESRGAN upscaling.\n", + "\u001b[K |████████████████████████████████| 776.3 MB 11 kB/s \n", + "\u001b[K |████████████████████████████████| 19.1 MB 22.8 MB/s \n", + "\u001b[?25hrunning develop\n", + "running egg_info\n", + "writing realesrgan.egg-info/PKG-INFO\n", + "writing dependency_links to realesrgan.egg-info/dependency_links.txt\n", + "writing requirements to realesrgan.egg-info/requires.txt\n", + "writing top-level names to realesrgan.egg-info/top_level.txt\n", + "reading manifest template 'MANIFEST.in'\n", + "adding license file 'LICENSE'\n", + "writing manifest file 'realesrgan.egg-info/SOURCES.txt'\n", + "running build_ext\n", + "Creating /usr/local/lib/python3.7/dist-packages/realesrgan.egg-link (link to .)\n", + "realesrgan 0.3.0 is already the active version in easy-install.pth\n", + "\n", + "Installed /content/gdrive/MyDrive/Real-ESRGAN\n", + "Processing dependencies for realesrgan==0.3.0\n", + "Searching for tqdm==4.64.1\n", + "Best match: tqdm 4.64.1\n", + "Adding tqdm 4.64.1 to easy-install.pth file\n", + "Installing tqdm script to /usr/local/bin\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for torchvision==0.13.1\n", + "Best match: torchvision 0.13.1\n", + "Adding torchvision 0.13.1 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for torch==1.12.1\n", + "Best match: torch 1.12.1\n", + "Adding torch 1.12.1 to easy-install.pth file\n", + "Installing convert-caffe2-to-onnx script to /usr/local/bin\n", + "Installing convert-onnx-to-caffe2 script to /usr/local/bin\n", + "Installing torchrun script to /usr/local/bin\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for Pillow==7.1.2\n", + "Best match: Pillow 7.1.2\n", + "Adding Pillow 7.1.2 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for opencv-python==4.6.0.66\n", + "Best match: opencv-python 4.6.0.66\n", + "Adding opencv-python 4.6.0.66 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for numpy==1.21.6\n", + "Best match: numpy 1.21.6\n", + "Adding numpy 1.21.6 to easy-install.pth file\n", + "Installing f2py script to /usr/local/bin\n", + "Installing f2py3 script to /usr/local/bin\n", + "Installing f2py3.7 script to /usr/local/bin\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for gfpgan==1.3.8\n", + "Best match: gfpgan 1.3.8\n", + "Adding gfpgan 1.3.8 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for facexlib==0.2.5\n", + "Best match: facexlib 0.2.5\n", + "Adding facexlib 0.2.5 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for basicsr==1.4.2\n", + "Best match: basicsr 1.4.2\n", + "Adding basicsr 1.4.2 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for typing-extensions==4.1.1\n", + "Best match: typing-extensions 4.1.1\n", + "Adding typing-extensions 4.1.1 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for requests==2.23.0\n", + "Best match: requests 2.23.0\n", + "Adding requests 2.23.0 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for yapf==0.32.0\n", + "Best match: yapf 0.32.0\n", + "Adding yapf 0.32.0 to easy-install.pth file\n", + "Installing yapf script to /usr/local/bin\n", + "Installing yapf-diff script to /usr/local/bin\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for scipy==1.7.3\n", + "Best match: scipy 1.7.3\n", + "Adding scipy 1.7.3 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for tb-nightly==2.11.0a20221002\n", + "Best match: tb-nightly 2.11.0a20221002\n", + "Adding tb-nightly 2.11.0a20221002 to easy-install.pth file\n", + "Installing tensorboard script to /usr/local/bin\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for lmdb==0.99\n", + "Best match: lmdb 0.99\n", + "Adding lmdb 0.99 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for PyYAML==6.0\n", + "Best match: PyYAML 6.0\n", + "Adding PyYAML 6.0 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for numba==0.56.2\n", + "Best match: numba 0.56.2\n", + "Adding numba 0.56.2 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for filterpy==1.4.5\n", + "Best match: filterpy 1.4.5\n", + "Adding filterpy 1.4.5 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for future==0.16.0\n", + "Best match: future 0.16.0\n", + "Adding future 0.16.0 to easy-install.pth file\n", + "Installing futurize script to /usr/local/bin\n", + "Installing pasteurize script to /usr/local/bin\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for addict==2.4.0\n", + "Best match: addict 2.4.0\n", + "Adding addict 2.4.0 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for scikit-image==0.18.3\n", + "Best match: scikit-image 0.18.3\n", + "Adding scikit-image 0.18.3 to easy-install.pth file\n", + "Installing skivi script to /usr/local/bin\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for idna==2.10\n", + "Best match: idna 2.10\n", + "Adding idna 2.10 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for chardet==3.0.4\n", + "Best match: chardet 3.0.4\n", + "Adding chardet 3.0.4 to easy-install.pth file\n", + "Installing chardetect script to /usr/local/bin\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for urllib3==1.24.3\n", + "Best match: urllib3 1.24.3\n", + "Adding urllib3 1.24.3 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for certifi==2022.6.15\n", + "Best match: certifi 2022.6.15\n", + "Adding certifi 2022.6.15 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for grpcio==1.48.1\n", + "Best match: grpcio 1.48.1\n", + "Adding grpcio 1.48.1 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for protobuf==3.17.3\n", + "Best match: protobuf 3.17.3\n", + "Adding protobuf 3.17.3 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for google-auth==1.35.0\n", + "Best match: google-auth 1.35.0\n", + "Adding google-auth 1.35.0 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for setuptools==57.4.0\n", + "Best match: setuptools 57.4.0\n", + "Adding setuptools 57.4.0 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for Werkzeug==1.0.1\n", + "Best match: Werkzeug 1.0.1\n", + "Adding Werkzeug 1.0.1 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for tensorboard-plugin-wit==1.8.1\n", + "Best match: tensorboard-plugin-wit 1.8.1\n", + "Adding tensorboard-plugin-wit 1.8.1 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for wheel==0.37.1\n", + "Best match: wheel 0.37.1\n", + "Adding wheel 0.37.1 to easy-install.pth file\n", + "Installing wheel script to /usr/local/bin\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for Markdown==3.4.1\n", + "Best match: Markdown 3.4.1\n", + "Adding Markdown 3.4.1 to easy-install.pth file\n", + "Installing markdown_py script to /usr/local/bin\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for google-auth-oauthlib==0.4.6\n", + "Best match: google-auth-oauthlib 0.4.6\n", + "Adding google-auth-oauthlib 0.4.6 to easy-install.pth file\n", + "Installing google-oauthlib-tool script to /usr/local/bin\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for absl-py==1.2.0\n", + "Best match: absl-py 1.2.0\n", + "Adding absl-py 1.2.0 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for tensorboard-data-server==0.6.1\n", + "Best match: tensorboard-data-server 0.6.1\n", + "Adding tensorboard-data-server 0.6.1 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for llvmlite==0.39.1\n", + "Best match: llvmlite 0.39.1\n", + "Adding llvmlite 0.39.1 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for importlib-metadata==4.12.0\n", + "Best match: importlib-metadata 4.12.0\n", + "Adding importlib-metadata 4.12.0 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for matplotlib==3.2.2\n", + "Best match: matplotlib 3.2.2\n", + "Adding matplotlib 3.2.2 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for networkx==2.6.3\n", + "Best match: networkx 2.6.3\n", + "Adding networkx 2.6.3 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for imageio==2.9.0\n", + "Best match: imageio 2.9.0\n", + "Adding imageio 2.9.0 to easy-install.pth file\n", + "Installing imageio_download_bin script to /usr/local/bin\n", + "Installing imageio_remove_bin script to /usr/local/bin\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for tifffile==2021.11.2\n", + "Best match: tifffile 2021.11.2\n", + "Adding tifffile 2021.11.2 to easy-install.pth file\n", + "Installing lsm2bin script to /usr/local/bin\n", + "Installing tiff2fsspec script to /usr/local/bin\n", + "Installing tiffcomment script to /usr/local/bin\n", + "Installing tifffile script to /usr/local/bin\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for PyWavelets==1.3.0\n", + "Best match: PyWavelets 1.3.0\n", + "Adding PyWavelets 1.3.0 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for six==1.15.0\n", + "Best match: six 1.15.0\n", + "Adding six 1.15.0 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for rsa==4.9\n", + "Best match: rsa 4.9\n", + "Adding rsa 4.9 to easy-install.pth file\n", + "Installing pyrsa-decrypt script to /usr/local/bin\n", + "Installing pyrsa-encrypt script to /usr/local/bin\n", + "Installing pyrsa-keygen script to /usr/local/bin\n", + "Installing pyrsa-priv2pub script to /usr/local/bin\n", + "Installing pyrsa-sign script to /usr/local/bin\n", + "Installing pyrsa-verify script to /usr/local/bin\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for pyasn1-modules==0.2.8\n", + "Best match: pyasn1-modules 0.2.8\n", + "Adding pyasn1-modules 0.2.8 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for cachetools==4.2.4\n", + "Best match: cachetools 4.2.4\n", + "Adding cachetools 4.2.4 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for requests-oauthlib==1.3.1\n", + "Best match: requests-oauthlib 1.3.1\n", + "Adding requests-oauthlib 1.3.1 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for zipp==3.8.1\n", + "Best match: zipp 3.8.1\n", + "Adding zipp 3.8.1 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for python-dateutil==2.8.2\n", + "Best match: python-dateutil 2.8.2\n", + "Adding python-dateutil 2.8.2 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for kiwisolver==1.4.4\n", + "Best match: kiwisolver 1.4.4\n", + "Adding kiwisolver 1.4.4 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for pyparsing==3.0.9\n", + "Best match: pyparsing 3.0.9\n", + "Adding pyparsing 3.0.9 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for cycler==0.11.0\n", + "Best match: cycler 0.11.0\n", + "Adding cycler 0.11.0 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for pyasn1==0.4.8\n", + "Best match: pyasn1 0.4.8\n", + "Adding pyasn1 0.4.8 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Searching for oauthlib==3.2.0\n", + "Best match: oauthlib 3.2.0\n", + "Adding oauthlib 3.2.0 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.7/dist-packages\n", + "Finished processing dependencies for realesrgan==0.3.0\n", + "Finished Installing libraries for Real-ESRGAN upscaling.\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [], + "metadata": { + "id": "uiBpqk8Bv4Sr" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rqiEtaOYXnrt" + }, + "source": [ + "# 5. Upscale and save upscaled video to x2_video_path\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "XTVL4m_zXqBU", + "outputId": "ed8063d5-84bb-4ab7-81da-88f41b4815ef" + }, + "source": [ + "print(\"Starting video upscale...\")\n", + "# ! python inference_realesrgan_video.py -i upload/onepiece_demo.mp4 -n realesr-animevideov3 -s 2 --suffix outx2\n", + "! python inference_realesrgan_video.py -i $video_path -n realesr-animevideov3 -s 2 --suffix outx2\n", + " # Argements\n", + " # -i, --input: input video\n", + " # -n, --model_name: Used model name\n", + " # -s, --outscale: Scale\n", + " # -suffix: Suffix of the output video\n", + "\n" + ], + "execution_count": 20, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "inference: 100% 909/909 [04:40<00:00, 3.24frame/s]\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + " # Move to specified path\n", + "\n", + "x2_workingfile_path = '/content/gdrive/MyDrive/Real-ESRGAN/results/*_outx2.mp4'\n", + "\n", + "!ls $x2_workingfile_path\n", + "!mv $x2_workingfile_path $x2_video_path " + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ytbpy0LXlcqz", + "outputId": "0853757d-fef2-4f54-8cae-2cc61df584f4" + }, + "execution_count": 29, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "/content/gdrive/MyDrive/Real-ESRGAN/results/Gs6_outx2.mp4\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "# Interpolate Frames with RIFE" + ], + "metadata": { + "id": "rNHQqP6ef3Qp" + } + }, + { + "cell_type": "code", + "source": [ + "## Install RIFE\n", + "\n", + "%cd /content\n", + "!git clone https://github.com/hzwer/arXiv2020-RIFE" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "MYxkLxiKfrDV", + "outputId": "c939822e-3ba3-45d6-ef1e-d6954a993682" + }, + "execution_count": 30, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "/content\n", + "fatal: destination path 'arXiv2020-RIFE' already exists and is not an empty directory.\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "## Download the interpolation AI model \n", + "\n", + "!mkdir /content/arXiv2020-RIFE/train_log\n", + "%cd /content/arXiv2020-RIFE/train_log\n", + "!gdown --id 1APIzVeI-4ZZCEuIRE1m6WYfSCaOsi_7_\n", + "!7z e -aoa RIFE_trained_model_v3.6.zip" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "yAhxuSRvf2co", + "outputId": "f6b40528-cbda-4761-ef69-3b6def3723ca" + }, + "execution_count": 31, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "mkdir: cannot create directory ‘/content/arXiv2020-RIFE/train_log’: File exists\n", + "/content/arXiv2020-RIFE/train_log\n", + "/usr/local/lib/python3.7/dist-packages/gdown/cli.py:131: FutureWarning: Option `--id` was deprecated in version 4.3.1 and will be removed in 5.0. You don't need to pass it anymore to use a file ID.\n", + " category=FutureWarning,\n", + "Downloading...\n", + "From: https://drive.google.com/uc?id=1APIzVeI-4ZZCEuIRE1m6WYfSCaOsi_7_\n", + "To: /content/arXiv2020-RIFE/train_log/RIFE_trained_model_v3.6.zip\n", + "100% 11.3M/11.3M [00:00<00:00, 291MB/s]\n", + "\n", + "7-Zip [64] 16.02 : Copyright (c) 1999-2016 Igor Pavlov : 2016-05-21\n", + "p7zip Version 16.02 (locale=en_US.UTF-8,Utf16=on,HugeFiles=on,64 bits,4 CPUs Intel(R) Xeon(R) CPU @ 2.20GHz (406F0),ASM,AES-NI)\n", + "\n", + "Scanning the drive for archives:\n", + " 0M Scan\b\b\b\b\b\b\b\b\b \b\b\b\b\b\b\b\b\b1 file, 11332064 bytes (11 MiB)\n", + "\n", + "Extracting archive: RIFE_trained_model_v3.6.zip\n", + "--\n", + "Path = RIFE_trained_model_v3.6.zip\n", + "Type = zip\n", + "Physical Size = 11332064\n", + "\n", + " 0%\b\b\b\b \b\b\b\b\n", + "Would you like to replace the existing file:\n", + " Path: ./.DS_Store\n", + " Size: 6148 bytes (7 KiB)\n", + " Modified: 2021-06-17 11:33:40\n", + "with the file from archive:\n", + " Path: train_log/.DS_Store\n", + " Size: 6148 bytes (7 KiB)\n", + " Modified: 2021-06-17 11:33:40\n", + "? (Y)es / (N)o / (A)lways / (S)kip all / A(u)to rename all / (Q)uit? A\n", + "\n", + " 0% 1 - train_log/.DS_Store\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b \b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\bEverything is Ok\n", + "\n", + "Folders: 4\n", + "Files: 10\n", + "Size: 12208819\n", + "Compressed: 11332064\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "%cd /content/arXiv2020-RIFE/\n", + "## Demo video not needed\n", + "## !gdown --id 1i3xlKb7ax7Y70khcTcuePi6E7crO_dFc\n", + "!pip3 -q install -r requirements.txt\n", + "print(\"You can safely ignore dependency errors for torchtext, torchaudio, gfpgan, fastai, basicsr and realesrgan - it works just fine without these...\")" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "tn2RZFPogBmi", + "outputId": "d3917412-98b8-4330-88cc-e9af09d4320c" + }, + "execution_count": 32, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "/content/arXiv2020-RIFE\n", + "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", + "torchtext 0.13.1 requires torch==1.12.1, but you have torch 1.6.0 which is incompatible.\n", + "torchaudio 0.12.1+cu113 requires torch==1.12.1, but you have torch 1.6.0 which is incompatible.\n", + "gfpgan 1.3.8 requires torch>=1.7, but you have torch 1.6.0 which is incompatible.\n", + "fastai 2.7.9 requires torch<1.14,>=1.7, but you have torch 1.6.0 which is incompatible.\n", + "fastai 2.7.9 requires torchvision>=0.8.2, but you have torchvision 0.7.0 which is incompatible.\n", + "basicsr 1.4.2 requires torch>=1.7, but you have torch 1.6.0 which is incompatible.\n", + "realesrgan 0.3.0 requires torch>=1.7, but you have torch 1.6.0 which is incompatible.\u001b[0m\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "!ls '/content/arXiv2020-RIFE/'\n", + "!cp $x2_video_path '/content/arXiv2020-RIFE/video.mp4'\n", + "!ls '/content/arXiv2020-RIFE/'\n" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "PsX8Ko0WgMaW", + "outputId": "6c279a88-94e5-407c-cd6c-ecd5fac27f5f" + }, + "execution_count": 35, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "benchmark\t docker\t model\t\ttrain.py\n", + "Colab_demo.ipynb inference_img.py README.md\n", + "dataset.py\t inference_video.py requirements.txt\n", + "demo\t\t LICENSE\t train_log\n", + "benchmark\t docker\t model\t\ttrain.py\n", + "Colab_demo.ipynb inference_img.py README.md\t\tvideo.mp4\n", + "dataset.py\t inference_video.py requirements.txt\n", + "demo\t\t LICENSE\t train_log\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "!python3 inference_video.py --help\n", + "\n", + "print(\"Starting video interpolation...\")\n", + "!python3 inference_video.py --exp=$interpolate_steps --video=video.mp4" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "cQbaiuQMiQ5B", + "outputId": "83431ec8-8b5f-4404-839f-d2481b907e42" + }, + "execution_count": 36, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Loaded v3.x HD model.\n", + "video.mp4, 909.0 frames in total, 5.0FPS to 40.0FPS\n", + "The audio will be merged after interpolation process\n", + "100% 908/909.0 [13:15<00:00, 1.14it/s]\n", + "\u001b[1;36m[mpeg4 @ 0x55b1600] \u001b[0m\u001b[1;31mInvalid pts (7264) <= last (7264)\n", + "\u001b[0m" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "!ls '/content/arXiv2020-RIFE/'\n", + "\n", + "## Copy back to gdrive\n", + "!cp /content/arXiv2020-RIFE/video*fps.mp4 $hifps_x2_video_path\n", + "\n", + "print(\"All done - if it worked, processed video should be available at: \" + hifps_x2_video_path)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "akrMO0JAihw1", + "outputId": "5aa1726b-85e2-4b6a-c907-4102ca9b34a4" + }, + "execution_count": 41, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "benchmark\t docker\t model\t\ttrain.py\n", + "Colab_demo.ipynb inference_img.py README.md\t\tvideo_8X_40fps.mp4\n", + "dataset.py\t inference_video.py requirements.txt\tvideo.mp4\n", + "demo\t\t LICENSE\t train_log\n", + "All done - if it worked, processed video should be available at: /content/gdrive/MyDrive/Gs6_x2_hifps.mp4\n" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/Silas_Video_Scale_and_Interpolate_Real_ESRGAN_(anime)_and_RIFE.ipynb b/Silas_Video_Scale_and_Interpolate_Real_ESRGAN_(anime)_and_RIFE.ipynb new file mode 100644 index 0000000..66774ba --- /dev/null +++ b/Silas_Video_Scale_and_Interpolate_Real_ESRGAN_(anime)_and_RIFE.ipynb @@ -0,0 +1,518 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [], + "collapsed_sections": [ + "6BPxh_VmVVIu" + ], + "machine_shape": "hm", + "include_colab_link": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IRDbDYYMQt_Y" + }, + "source": [ + "# Video upscaling and frame interpolation tool\n", + "\n", + "There was nothing super reliable available, so I cobbled this together.\n", + "\n", + "Tested on colab+ / google drive. You'll need both of these for it to work, probably.\n", + "\n", + "From:\n", + "- Real-ESRGAN [GitHub repo](https://github.com/xinntao/Real-ESRGAN). \n", + "- RIFE [GitHub repo](https://github.com/megvii-research/ECCV2022-RIFE).\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6BPxh_VmVVIu" + }, + "source": [ + "# ESRGAN prep\n", + "Before start, make sure that you choose\n", + "* Runtime Type = Python 3\n", + "* Hardware Accelerator = GPU\n", + "\n", + "in the **Runtime** menu -> **Change runtime type**\n", + "\n", + "Then, we clone the repository, set up the envrironment." + ] + }, + { + "cell_type": "code", + "source": [ + "## See what GPU you got\n", + "!nvidia-smi" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "RQJgGjNXjolo", + "outputId": "e9458f7b-9fa5-4955-d9b1-8276154dfae2" + }, + "execution_count": 1, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Mon Oct 3 21:54:21 2022 \n", + "+-----------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 460.32.03 Driver Version: 460.32.03 CUDA Version: 11.2 |\n", + "|-------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|===============================+======================+======================|\n", + "| 0 Tesla P100-PCIE... Off | 00000000:00:04.0 Off | 0 |\n", + "| N/A 37C P0 27W / 250W | 0MiB / 16280MiB | 0% Default |\n", + "| | | N/A |\n", + "+-------------------------------+----------------------+----------------------+\n", + " \n", + "+-----------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=============================================================================|\n", + "| No running processes found |\n", + "+-----------------------------------------------------------------------------+\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "#@markdown #Parameters\n", + "#@markdown ###Filenames and paths\n", + "working_dir = '/content/gdrive/MyDrive/' #@param {type:\"string\"}\n", + "original_video = 'GS7.mp4' #@param {type:\"string\"}\n", + "x2_video = 'GS7_x2.mp4' #@param {type:\"string\"}\n", + "hifps_x2_video = 'GS7_x2_hifps.mp4' #@param {type:\"string\"}\n", + "\n", + "#@markdown ###
Scale the video up by this much\n", + "video_scale=3#@param {type:\"slider\", min:1, max:5, step:1}\n", + "\n", + "#@markdown ####
Interpolate steps: 1=2x, 2=4x, 3=8x, 4=16x (approx)\n", + "#@markdown For example, 3 will convert a 5fps video to 40fps.\n", + "interpolate_steps=3#@param {type:\"slider\", min:1, max:5, step:1}\n", + "\n", + "\n", + "## video_path = 'upload/Gs6.mp4' #@param {type:\"string\"}\n", + "\n", + "video_path = working_dir + original_video\n", + "x2_video_path = working_dir + x2_video\n", + "hifps_x2_video_path = working_dir + hifps_x2_video\n", + "\n", + "\n", + "#Mount Google Drive\n", + "from google.colab import drive\n", + "drive.mount('/content/gdrive')" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "5E3mWzgcny0l", + "outputId": "3e262f32-4185-4ca7-c37e-3960249427fe" + }, + "execution_count": 2, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Mounted at /content/gdrive\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "GnpnrLfMV2jU", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "e167d652-b090-407e-f4e7-813688a8beba" + }, + "source": [ + "# Clone Real-ESRGAN, install dependencies\n", + "\n", + "# Clone to gdrive if not there already\n", + "import os\n", + "if os.path.isdir('/content/gdrive/MyDrive/Real-ESRGAN'):\n", + " %cd /content/gdrive/MyDrive/Real-ESRGAN\n", + "elif os.path.isdir('/content/gdrive/MyDrive/'):\n", + " %cd /content/gdrive/MyDrive\n", + " !git clone https://github.com/xinntao/Real-ESRGAN.git\n", + " %cd Real-ESRGAN\n", + "\n", + "# Set up the environment\n", + "print(\"Started Installing libraries for Real-ESRGAN upscaling.\")\n", + "!pip -q install basicsr\n", + "!pip -q install facexlib\n", + "!pip -q install gfpgan\n", + "!pip -q install ffmpeg-python\n", + "!pip -q install -r requirements.txt\n", + "print(\"Running setup.py - this will take a while...\")\n", + "!python setup.py develop &> /dev/null\n", + "print(\"Finished Installing libraries for Real-ESRGAN upscaling.\")" + ], + "execution_count": 3, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "/content/gdrive/MyDrive/Real-ESRGAN\n", + "Started Installing libraries for Real-ESRGAN upscaling.\n", + "\u001b[K |████████████████████████████████| 172 kB 7.3 MB/s \n", + "\u001b[K |████████████████████████████████| 6.0 MB 74.8 MB/s \n", + "\u001b[K |████████████████████████████████| 190 kB 95.9 MB/s \n", + "\u001b[?25h Building wheel for basicsr (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "\u001b[K |████████████████████████████████| 59 kB 4.3 MB/s \n", + "\u001b[K |████████████████████████████████| 177 kB 15.8 MB/s \n", + "\u001b[?25h Building wheel for filterpy (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "\u001b[K |████████████████████████████████| 52 kB 937 kB/s \n", + "\u001b[?25hRunning setup.py - this will take a while...\n", + "Finished Installing libraries for Real-ESRGAN upscaling.\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rqiEtaOYXnrt" + }, + "source": [ + "# 5. Upscale and save upscaled video to x2_video_path\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "XTVL4m_zXqBU", + "outputId": "ff128ffc-e392-4297-b1f1-c5d65e5f466e" + }, + "source": [ + "print(\"Starting video upscale...\")\n", + "# ! python inference_realesrgan_video.py -i upload/onepiece_demo.mp4 -n realesr-animevideov3 -s 2 --suffix outx2\n", + "! python inference_realesrgan_video.py -i $video_path -n realesr-animevideov3 -s $video_scale --suffix outx2\n", + " # Argements\n", + " # -i, --input: input video\n", + " # -n, --model_name: Used model name\n", + " # -s, --outscale: Scale\n", + " # -suffix: Suffix of the output video\n", + "\n" + ], + "execution_count": 4, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Starting video upscale...\n", + "inference: 100% 919/919 [06:27<00:00, 2.37frame/s]\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + " # Move to specified path\n", + "\n", + "x2_workingfile_path = '/content/gdrive/MyDrive/Real-ESRGAN/results/*_outx2.mp4'\n", + "\n", + "!ls $x2_workingfile_path\n", + "!mv $x2_workingfile_path $x2_video_path " + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ytbpy0LXlcqz", + "outputId": "17019f29-a8b1-47f4-c965-c43d64693435" + }, + "execution_count": 5, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "/content/gdrive/MyDrive/Real-ESRGAN/results/GS7_outx2.mp4\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "# Interpolate Frames with RIFE" + ], + "metadata": { + "id": "rNHQqP6ef3Qp" + } + }, + { + "cell_type": "code", + "source": [ + "## Install RIFE\n", + "\n", + "%cd /content\n", + "!git clone https://github.com/hzwer/arXiv2020-RIFE" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "MYxkLxiKfrDV", + "outputId": "d73482f1-5539-46ff-bd24-158f7d69908b" + }, + "execution_count": 6, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "/content\n", + "Cloning into 'arXiv2020-RIFE'...\n", + "remote: Enumerating objects: 1888, done.\u001b[K\n", + "remote: Counting objects: 100% (63/63), done.\u001b[K\n", + "remote: Compressing objects: 100% (44/44), done.\u001b[K\n", + "remote: Total 1888 (delta 34), reused 43 (delta 19), pack-reused 1825\u001b[K\n", + "Receiving objects: 100% (1888/1888), 4.08 MiB | 1.08 MiB/s, done.\n", + "Resolving deltas: 100% (1187/1187), done.\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "## Download the interpolation AI model \n", + "\n", + "!mkdir /content/arXiv2020-RIFE/train_log\n", + "%cd /content/arXiv2020-RIFE/train_log\n", + "!gdown --id 1APIzVeI-4ZZCEuIRE1m6WYfSCaOsi_7_\n", + "!7z e -aoa RIFE_trained_model_v3.6.zip" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "yAhxuSRvf2co", + "outputId": "11861543-4fb2-42c2-90fe-ad3b4688307a" + }, + "execution_count": 7, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "/content/arXiv2020-RIFE/train_log\n", + "/usr/local/lib/python3.7/dist-packages/gdown/cli.py:131: FutureWarning: Option `--id` was deprecated in version 4.3.1 and will be removed in 5.0. You don't need to pass it anymore to use a file ID.\n", + " category=FutureWarning,\n", + "Downloading...\n", + "From: https://drive.google.com/uc?id=1APIzVeI-4ZZCEuIRE1m6WYfSCaOsi_7_\n", + "To: /content/arXiv2020-RIFE/train_log/RIFE_trained_model_v3.6.zip\n", + "100% 11.3M/11.3M [00:00<00:00, 54.2MB/s]\n", + "\n", + "7-Zip [64] 16.02 : Copyright (c) 1999-2016 Igor Pavlov : 2016-05-21\n", + "p7zip Version 16.02 (locale=en_US.UTF-8,Utf16=on,HugeFiles=on,64 bits,4 CPUs Intel(R) Xeon(R) CPU @ 2.20GHz (406F0),ASM,AES-NI)\n", + "\n", + "Scanning the drive for archives:\n", + " 0M Scan\b\b\b\b\b\b\b\b\b \b\b\b\b\b\b\b\b\b1 file, 11332064 bytes (11 MiB)\n", + "\n", + "Extracting archive: RIFE_trained_model_v3.6.zip\n", + "--\n", + "Path = RIFE_trained_model_v3.6.zip\n", + "Type = zip\n", + "Physical Size = 11332064\n", + "\n", + " 0%\b\b\b\b \b\b\b\bEverything is Ok\n", + "\n", + "Folders: 4\n", + "Files: 10\n", + "Size: 12208819\n", + "Compressed: 11332064\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "%cd /content/arXiv2020-RIFE/\n", + "## Demo video not needed\n", + "## !gdown --id 1i3xlKb7ax7Y70khcTcuePi6E7crO_dFc\n", + "!pip3 -q install -r requirements.txt\n", + "print(\"You can safely ignore dependency errors for torchtext, torchaudio, gfpgan, fastai, basicsr and realesrgan - it works just fine without these...\")" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "tn2RZFPogBmi", + "outputId": "fe06b7a2-1219-4b99-fcc7-143df67dea5f" + }, + "execution_count": 8, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "/content/arXiv2020-RIFE\n", + "\u001b[K |████████████████████████████████| 2.3 MB 6.5 MB/s \n", + "\u001b[K |████████████████████████████████| 748.8 MB 18 kB/s \n", + "\u001b[K |████████████████████████████████| 388 kB 92.5 MB/s \n", + "\u001b[K |████████████████████████████████| 5.9 MB 57.0 MB/s \n", + "\u001b[K |████████████████████████████████| 26.9 MB 1.2 MB/s \n", + "\u001b[?25h Building wheel for moviepy (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", + "torchtext 0.13.1 requires torch==1.12.1, but you have torch 1.6.0 which is incompatible.\n", + "torchaudio 0.12.1+cu113 requires torch==1.12.1, but you have torch 1.6.0 which is incompatible.\n", + "gfpgan 1.3.8 requires torch>=1.7, but you have torch 1.6.0 which is incompatible.\n", + "fastai 2.7.9 requires torch<1.14,>=1.7, but you have torch 1.6.0 which is incompatible.\n", + "fastai 2.7.9 requires torchvision>=0.8.2, but you have torchvision 0.7.0 which is incompatible.\n", + "basicsr 1.4.2 requires torch>=1.7, but you have torch 1.6.0 which is incompatible.\n", + "realesrgan 0.3.0 requires torch>=1.7, but you have torch 1.6.0 which is incompatible.\u001b[0m\n", + "You can safely ignore dependency errors for torchtext, torchaudio, gfpgan, fastai, basicsr and realesrgan - it works just fine without these...\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "!ls '/content/arXiv2020-RIFE/'\n", + "!cp $x2_video_path '/content/arXiv2020-RIFE/video.mp4'\n", + "!ls '/content/arXiv2020-RIFE/'\n" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "PsX8Ko0WgMaW", + "outputId": "4ff1ccfd-8363-48cf-adcf-ecd2b7908898" + }, + "execution_count": 9, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "benchmark\t docker\t model\t\ttrain.py\n", + "Colab_demo.ipynb inference_img.py README.md\n", + "dataset.py\t inference_video.py requirements.txt\n", + "demo\t\t LICENSE\t train_log\n", + "benchmark\t docker\t model\t\ttrain.py\n", + "Colab_demo.ipynb inference_img.py README.md\t\tvideo.mp4\n", + "dataset.py\t inference_video.py requirements.txt\n", + "demo\t\t LICENSE\t train_log\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "!python3 inference_video.py --help\n", + "\n", + "print(\"Starting video interpolation...\")\n", + "!python3 inference_video.py --exp=$interpolate_steps --video=video.mp4" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "cQbaiuQMiQ5B", + "outputId": "f617b75a-d09e-4e03-8316-bb914a3126de" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "usage: inference_video.py [-h] [--video VIDEO] [--output OUTPUT] [--img IMG]\n", + " [--montage] [--model MODELDIR] [--fp16] [--UHD]\n", + " [--scale SCALE] [--skip] [--fps FPS] [--png]\n", + " [--ext EXT] [--exp EXP]\n", + "\n", + "Interpolation for a pair of images\n", + "\n", + "optional arguments:\n", + " -h, --help show this help message and exit\n", + " --video VIDEO\n", + " --output OUTPUT\n", + " --img IMG\n", + " --montage montage origin video\n", + " --model MODELDIR directory with trained model files\n", + " --fp16 fp16 mode for faster and more lightweight inference on\n", + " cards with Tensor Cores\n", + " --UHD support 4k video\n", + " --scale SCALE Try scale=0.5 for 4k video\n", + " --skip whether to remove static frames before processing\n", + " --fps FPS\n", + " --png whether to vid_out png format vid_outs\n", + " --ext EXT vid_out video extension\n", + " --exp EXP\n", + "Starting video interpolation...\n", + "Loaded v3.x HD model.\n", + "video.mp4, 919.0 frames in total, 5.0FPS to 40.0FPS\n", + "The audio will be merged after interpolation process\n", + " 9% 80/919.0 [02:38<26:59, 1.93s/it]" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "!ls '/content/arXiv2020-RIFE/'\n", + "\n", + "## Copy back to gdrive\n", + "!cp /content/arXiv2020-RIFE/video*fps.mp4 $hifps_x2_video_path\n", + "\n", + "print(\"All done - if it worked, processed video should be available at: \" + hifps_x2_video_path)" + ], + "metadata": { + "id": "akrMO0JAihw1" + }, + "execution_count": null, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/app/code/community/Bolevar/AdaptiveResize/Helper/Image.php b/app/code/community/Bolevar/AdaptiveResize/Helper/Image.php index 4028b8c..6e02f27 100644 --- a/app/code/community/Bolevar/AdaptiveResize/Helper/Image.php +++ b/app/code/community/Bolevar/AdaptiveResize/Helper/Image.php @@ -1,5 +1,4 @@ getSkinUrl($this->getPlaceholder()); + + $file = Mage::getDesign()->getSkinUrl($this->getPlaceholder()); + + if ($this->getImageFile() == $file) + { + $url = $file; + } + else + { + $this->setImageFile($file); + $url = $this->__toString(); + } + } return $url; } diff --git a/lib/Varien/Image/Adapter/Gd2.php b/lib/Varien/Image/Adapter/Gd2.php new file mode 100644 index 0000000..5bbf7bc --- /dev/null +++ b/lib/Varien/Image/Adapter/Gd2.php @@ -0,0 +1,599 @@ + array('output' => 'imagegif', 'create' => 'imagecreatefromgif'), + IMAGETYPE_JPEG => array('output' => 'imagejpeg', 'create' => 'imagecreatefromjpeg'), + IMAGETYPE_PNG => array('output' => 'imagepng', 'create' => 'imagecreatefrompng'), + IMAGETYPE_XBM => array('output' => 'imagexbm', 'create' => 'imagecreatefromxbm'), + IMAGETYPE_WBMP => array('output' => 'imagewbmp', 'create' => 'imagecreatefromxbm'), + ); + + /** + * Whether image was resized or not + * + * @var bool + */ + protected $_resized = false; + + /** + * Opens image file. + * + * @param string $filename + * @throws Varien_Exception + */ + public function open($filename) + { + $this->_fileName = $filename; + $this->getMimeType(); + $this->_getFileAttributes(); + if ($this->_isMemoryLimitReached()) { + throw new Varien_Exception('Memory limit has been reached.'); + } + $this->_imageHandler = call_user_func($this->_getCallback('create'), $this->_fileName); + } + + /** + * Checks whether memory limit is reached. + * + * @return bool + */ + protected function _isMemoryLimitReached() + { + $limit = $this->_convertToByte(ini_get('memory_limit')); + $size = getimagesize($this->_fileName); + $requiredMemory = $size[0] * $size[1] * 3; + return (memory_get_usage(true) + $requiredMemory) > $limit; + } + + /** + * Converts memory value (e.g. 64M, 129KB) to bytes. + * Case insensitive value might be used. + * + * @param string $memoryValue + * @return int + */ + protected function _convertToByte($memoryValue) + { + + if (stripos($memoryValue, 'M') !== false) { + return (int)$memoryValue * 1024 * 1024; + } elseif (stripos($memoryValue, 'KB') !== false) { + return (int)$memoryValue * 1024; + } elseif (stripos($memoryValue, 'G') !== false) { + return (int)$memoryValue * 1024 * 1024 * 1024; + } + + return (int)$memoryValue; + } + + public function save($destination=null, $newName=null) + { + $fileName = ( !isset($destination) ) ? $this->_fileName : $destination; + + if( isset($destination) && isset($newName) ) { + $fileName = $destination . "/" . $newName; + } elseif( isset($destination) && !isset($newName) ) { + $info = pathinfo($destination); + $fileName = $destination; + $destination = $info['dirname']; + } elseif( !isset($destination) && isset($newName) ) { + $fileName = $this->_fileSrcPath . "/" . $newName; + } else { + $fileName = $this->_fileSrcPath . $this->_fileSrcName; + } + + $destinationDir = ( isset($destination) ) ? $destination : $this->_fileSrcPath; + + if( !is_writable($destinationDir) ) { + try { + $io = new Varien_Io_File(); + $io->mkdir($destination); + } catch (Exception $e) { + throw new Exception("Unable to write file into directory '{$destinationDir}'. Access forbidden."); + } + } + + if (!$this->_resized) { + // keep alpha transparency + $isAlpha = false; + $isTrueColor = false; + $this->_getTransparency($this->_imageHandler, $this->_fileType, $isAlpha, $isTrueColor); + if ($isAlpha) { + if ($isTrueColor) { + $newImage = imagecreatetruecolor($this->_imageSrcWidth, $this->_imageSrcHeight); + } else { + $newImage = imagecreate($this->_imageSrcWidth, $this->_imageSrcHeight); + } + $this->_fillBackgroundColor($newImage); + imagecopy( + $newImage, + $this->_imageHandler, + 0, 0, + 0, 0, + $this->_imageSrcWidth, $this->_imageSrcHeight + ); + $this->_imageHandler = $newImage; + } + } + + $functionParameters = array(); + $functionParameters[] = $this->_imageHandler; + $functionParameters[] = $fileName; + + // set quality param for JPG file type + if (!is_null($this->quality()) && $this->_fileType == IMAGETYPE_JPEG) + { + $functionParameters[] = $this->quality(); + } + + // set quality param for PNG file type + if (!is_null($this->quality()) && $this->_fileType == IMAGETYPE_PNG) + { + $quality = round(($this->quality() / 100) * 10); + if ($quality < 1) { + $quality = 1; + } elseif ($quality > 10) { + $quality = 10; + } + $quality = 10 - $quality; + $functionParameters[] = $quality; + } + + call_user_func_array($this->_getCallback('output'), $functionParameters); + } + + public function display() + { + header("Content-type: ".$this->getMimeType()); + call_user_func($this->_getCallback('output'), $this->_imageHandler); + } + + /** + * Obtain function name, basing on image type and callback type + * + * @param string $callbackType + * @param int $fileType + * @return string + * @throws Exception + */ + private function _getCallback($callbackType, $fileType = null, $unsupportedText = 'Unsupported image format.') + { + if (null === $fileType) { + $fileType = $this->_fileType; + } + if (empty(self::$_callbacks[$fileType])) { + throw new Exception($unsupportedText); + } + if (empty(self::$_callbacks[$fileType][$callbackType])) { + throw new Exception('Callback not found.'); + } + return self::$_callbacks[$fileType][$callbackType]; + } + + private function _fillBackgroundColor(&$imageResourceTo) + { + // try to keep transparency, if any + if ($this->_keepTransparency) { + $isAlpha = false; + $transparentIndex = $this->_getTransparency($this->_imageHandler, $this->_fileType, $isAlpha); + try { + // fill truecolor png with alpha transparency + if ($isAlpha) { + + if (!imagealphablending($imageResourceTo, false)) { + throw new Exception('Failed to set alpha blending for PNG image.'); + } + $transparentAlphaColor = imagecolorallocatealpha($imageResourceTo, 0, 0, 0, 127); + if (false === $transparentAlphaColor) { + throw new Exception('Failed to allocate alpha transparency for PNG image.'); + } + if (!imagefill($imageResourceTo, 0, 0, $transparentAlphaColor)) { + throw new Exception('Failed to fill PNG image with alpha transparency.'); + } + if (!imagesavealpha($imageResourceTo, true)) { + throw new Exception('Failed to save alpha transparency into PNG image.'); + } + + return $transparentAlphaColor; + } + // fill image with indexed non-alpha transparency + elseif (false !== $transparentIndex) { + $transparentColor = false; + if ($transparentIndex >=0 && $transparentIndex <= imagecolorstotal($this->_imageHandler)) { + list($r, $g, $b) = array_values(imagecolorsforindex($this->_imageHandler, $transparentIndex)); + $transparentColor = imagecolorallocate($imageResourceTo, $r, $g, $b); + } + if (false === $transparentColor) { + throw new Exception('Failed to allocate transparent color for image.'); + } + if (!imagefill($imageResourceTo, 0, 0, $transparentColor)) { + throw new Exception('Failed to fill image with transparency.'); + } + imagecolortransparent($imageResourceTo, $transparentColor); + return $transparentColor; + } + } + catch (Exception $e) { + // fallback to default background color + } + } + list($r, $g, $b) = $this->_backgroundColor; + $color = imagecolorallocate($imageResourceTo, $r, $g, $b); + if (!imagefill($imageResourceTo, 0, 0, $color)) { + throw new Exception("Failed to fill image background with color {$r} {$g} {$b}."); + } + + return $color; + } + + /** + * Gives true for a PNG with alpha, false otherwise + * + * @param string $fileName + * @return boolean + */ + + public function checkAlpha($fileName) + { + return ((ord(file_get_contents($fileName, false, null, 25, 1)) & 6) & 4) == 4; + } + + private function _getTransparency($imageResource, $fileType, &$isAlpha = false, &$isTrueColor = false) + { + $isAlpha = false; + $isTrueColor = false; + // assume that transparency is supported by gif/png only + if ((IMAGETYPE_GIF === $fileType) || (IMAGETYPE_PNG === $fileType)) { + // check for specific transparent color + $transparentIndex = imagecolortransparent($imageResource); + if ($transparentIndex >= 0) { + return $transparentIndex; + } + // assume that truecolor PNG has transparency + elseif (IMAGETYPE_PNG === $fileType) { + $isAlpha = $this->checkAlpha($this->_fileName); + $isTrueColor = true; + return $transparentIndex; // -1 + } + } + if (IMAGETYPE_JPEG === $fileType) { + $isTrueColor = true; + } + return false; + } + + /** + * Change the image size + * + * @param int $frameWidth + * @param int $frameHeight + */ + public function resize($frameWidth = null, $frameHeight = null) + { + if (empty($frameWidth) && empty($frameHeight)) { + throw new Exception('Invalid image dimensions.'); + } + + // calculate lacking dimension + if (!$this->_keepFrame) { + if (null === $frameWidth) { + $frameWidth = round($frameHeight * ($this->_imageSrcWidth / $this->_imageSrcHeight)); + } + elseif (null === $frameHeight) { + $frameHeight = round($frameWidth * ($this->_imageSrcHeight / $this->_imageSrcWidth)); + } + } + else { + if (null === $frameWidth) { + $frameWidth = $frameHeight; + } + elseif (null === $frameHeight) { + $frameHeight = $frameWidth; + } + } + + // define coordinates of image inside new frame + $srcX = 0; + $srcY = 0; + $dstX = 0; + $dstY = 0; + $dstWidth = $frameWidth; + $dstHeight = $frameHeight; + if ($this->_keepAspectRatio) { + // do not make picture bigger, than it is, if required + if ($this->_constrainOnly) { + if (($frameWidth >= $this->_imageSrcWidth) && ($frameHeight >= $this->_imageSrcHeight)) { + $dstWidth = $this->_imageSrcWidth; + $dstHeight = $this->_imageSrcHeight; + } + } + // keep aspect ratio + if ($this->_imageSrcWidth / $this->_imageSrcHeight >= $frameWidth / $frameHeight) { + $dstHeight = round(($dstWidth / $this->_imageSrcWidth) * $this->_imageSrcHeight); + } else { + $dstWidth = round(($dstHeight / $this->_imageSrcHeight) * $this->_imageSrcWidth); + } + } + // define position in center (TODO: add positions option) + $dstY = round(($frameHeight - $dstHeight) / 2); + $dstX = round(($frameWidth - $dstWidth) / 2); + + // get rid of frame (fallback to zero position coordinates) + if (!$this->_keepFrame) { + $frameWidth = $dstWidth; + $frameHeight = $dstHeight; + $dstY = 0; + $dstX = 0; + } + + // create new image + $isAlpha = false; + $isTrueColor = false; + $this->_getTransparency($this->_imageHandler, $this->_fileType, $isAlpha, $isTrueColor); + if ($isTrueColor) { + $newImage = imagecreatetruecolor($frameWidth, $frameHeight); + } + else { + $newImage = imagecreate($frameWidth, $frameHeight); + } + + // fill new image with required color + $this->_fillBackgroundColor($newImage); + + // resample source image and copy it into new frame + imagecopyresampled( + $newImage, + $this->_imageHandler, + $dstX, $dstY, + $srcX, $srcY, + $dstWidth, $dstHeight, + $this->_imageSrcWidth, $this->_imageSrcHeight + ); + $this->_imageHandler = $newImage; + $this->refreshImageDimensions(); + $this->_resized = true; + } + + public function rotate($angle) + { +/* + $isAlpha = false; + $backgroundColor = $this->_getTransparency($this->_imageHandler, $this->_fileType, $isAlpha); + list($r, $g, $b) = $this->_backgroundColor; + if ($isAlpha) { + $backgroundColor = imagecolorallocatealpha($this->_imageHandler, 0, 0, 0, 127); + } + elseif (false === $backgroundColor) { + $backgroundColor = imagecolorallocate($this->_imageHandler, $r, $g, $b); + } + $this->_imageHandler = imagerotate($this->_imageHandler, $angle, $backgroundColor); +//*/ + $this->_imageHandler = imagerotate($this->_imageHandler, $angle, $this->imageBackgroundColor); + $this->refreshImageDimensions(); + } + + public function watermark($watermarkImage, $positionX=0, $positionY=0, $watermarkImageOpacity=30, $repeat=false) + { + list($watermarkSrcWidth, $watermarkSrcHeight, $watermarkFileType, ) = getimagesize($watermarkImage); + $this->_getFileAttributes(); + $watermark = call_user_func($this->_getCallback( + 'create', + $watermarkFileType, + 'Unsupported watermark image format.' + ), $watermarkImage); + + $merged = false; + + if ($this->getWatermarkWidth() && + $this->getWatermarkHeigth() && + ($this->getWatermarkPosition() != self::POSITION_STRETCH) + ) { + $newWatermark = imagecreatetruecolor($this->getWatermarkWidth(), $this->getWatermarkHeigth()); + imagealphablending($newWatermark, false); + $col = imagecolorallocate($newWatermark, 255, 255, 255); + imagecolortransparent($newWatermark, $col); + imagefilledrectangle($newWatermark, 0, 0, $this->getWatermarkWidth(), $this->getWatermarkHeigth(), $col); + imagealphablending($newWatermark, true); + imageSaveAlpha($newWatermark, true); + imagecopyresampled( + $newWatermark, + $watermark, + 0, 0, 0, 0, + $this->getWatermarkWidth(), $this->getWatermarkHeigth(), + imagesx($watermark), imagesy($watermark) + ); + $watermark = $newWatermark; + } + + if( $this->getWatermarkPosition() == self::POSITION_TILE ) { + $repeat = true; + } elseif( $this->getWatermarkPosition() == self::POSITION_STRETCH ) { + + $newWatermark = imagecreatetruecolor($this->_imageSrcWidth, $this->_imageSrcHeight); + imagealphablending($newWatermark, false); + $col = imagecolorallocate($newWatermark, 255, 255, 255); + imagecolortransparent($newWatermark, $col); + imagefilledrectangle($newWatermark, 0, 0, $this->_imageSrcWidth, $this->_imageSrcHeight, $col); + imagealphablending($newWatermark, true); + imageSaveAlpha($newWatermark, true); + imagecopyresampled( + $newWatermark, + $watermark, + 0, 0, 0, 0, + $this->_imageSrcWidth, $this->_imageSrcHeight, + imagesx($watermark), imagesy($watermark) + ); + $watermark = $newWatermark; + + } elseif( $this->getWatermarkPosition() == self::POSITION_CENTER ) { + $positionX = ($this->_imageSrcWidth/2 - imagesx($watermark)/2); + $positionY = ($this->_imageSrcHeight/2 - imagesy($watermark)/2); + imagecopymerge( + $this->_imageHandler, + $watermark, + $positionX, $positionY, + 0, 0, + imagesx($watermark), imagesy($watermark), + $this->getWatermarkImageOpacity() + ); + } elseif( $this->getWatermarkPosition() == self::POSITION_TOP_RIGHT ) { + $positionX = ($this->_imageSrcWidth - imagesx($watermark)); + imagecopymerge( + $this->_imageHandler, + $watermark, + $positionX, $positionY, + 0, 0, + imagesx($watermark), imagesy($watermark), + $this->getWatermarkImageOpacity() + ); + } elseif( $this->getWatermarkPosition() == self::POSITION_TOP_LEFT ) { + imagecopymerge( + $this->_imageHandler, + $watermark, + $positionX, $positionY, + 0, 0, + imagesx($watermark), imagesy($watermark), + $this->getWatermarkImageOpacity() + ); + } elseif( $this->getWatermarkPosition() == self::POSITION_BOTTOM_RIGHT ) { + $positionX = ($this->_imageSrcWidth - imagesx($watermark)); + $positionY = ($this->_imageSrcHeight - imagesy($watermark)); + imagecopymerge( + $this->_imageHandler, + $watermark, + $positionX, $positionY, + 0, 0, + imagesx($watermark), imagesy($watermark), + $this->getWatermarkImageOpacity() + ); + } elseif( $this->getWatermarkPosition() == self::POSITION_BOTTOM_LEFT ) { + $positionY = ($this->_imageSrcHeight - imagesy($watermark)); + imagecopymerge( + $this->_imageHandler, + $watermark, + $positionX, $positionY, + 0, 0, + imagesx($watermark), imagesy($watermark), + $this->getWatermarkImageOpacity() + ); + } + + if( $repeat === false && $merged === false ) { + imagecopymerge( + $this->_imageHandler, + $watermark, + $positionX, $positionY, + 0, 0, + imagesx($watermark), imagesy($watermark), + $this->getWatermarkImageOpacity() + ); + } else { + $offsetX = $positionX; + $offsetY = $positionY; + while( $offsetY <= ($this->_imageSrcHeight+imagesy($watermark)) ) { + while( $offsetX <= ($this->_imageSrcWidth+imagesx($watermark)) ) { + imagecopymerge( + $this->_imageHandler, + $watermark, + $offsetX, $offsetY, + 0, 0, + imagesx($watermark), imagesy($watermark), + $this->getWatermarkImageOpacity() + ); + $offsetX += imagesx($watermark); + } + $offsetX = $positionX; + $offsetY += imagesy($watermark); + } + } + + imagedestroy($watermark); + $this->refreshImageDimensions(); + } + + public function crop($top=0, $left=0, $right=0, $bottom=0) + { + if( $left == 0 && $top == 0 && $right == 0 && $bottom == 0 ) { + return; + } + + $newWidth = $this->_imageSrcWidth - $left - $right; + $newHeight = $this->_imageSrcHeight - $top - $bottom; + + $canvas = imagecreatetruecolor($newWidth, $newHeight); + + if ($this->_fileType == IMAGETYPE_PNG) { + $this->_saveAlpha($canvas); + } + + imagecopyresampled( + $canvas, + $this->_imageHandler, + 0, 0, $left, $top, + $newWidth, $newHeight, + $newWidth, $newHeight + ); + + $this->_imageHandler = $canvas; + $this->refreshImageDimensions(); + } + + public function checkDependencies() + { + foreach( $this->_requiredExtensions as $value ) { + if( !extension_loaded($value) ) { + throw new Exception("Required PHP extension '{$value}' was not loaded."); + } + } + } + + private function refreshImageDimensions() + { + $this->_imageSrcWidth = imagesx($this->_imageHandler); + $this->_imageSrcHeight = imagesy($this->_imageHandler); + } + + function __destruct() + { + @imagedestroy($this->_imageHandler); + } + + /* + * Fixes saving PNG alpha channel + */ + private function _saveAlpha($imageHandler) + { + $background = imagecolorallocate($imageHandler, 0, 0, 0); + ImageColorTransparent($imageHandler, $background); + imagealphablending($imageHandler, false); + imagesavealpha($imageHandler, true); + } +}