inference.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When running this file, you need to modify the device that loads each model and the parameters of the calling function according to your own needs."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Video Foundation Model\n",
    "\n",
    "### Video Understanding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/temp/miniconda/envs/dolphin/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing mPLUG for VideoCaptioning\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/temp/miniconda/envs/dolphin/lib/python3.8/site-packages/transformers/configuration_utils.py:379: UserWarning: Passing `gradient_checkpointing` to a config initialization is deprecated and will be removed in v5 Transformers. Using `model.gradient_checkpointing_enable()` instead, or if you are using the `Trainer` API, pass `gradient_checkpointing=True` in your `TrainingArguments`.\n",
      "  warnings.warn(\n",
      "/temp/miniconda/envs/dolphin/lib/python3.8/site-packages/torchvision/transforms/transforms.py:332: UserWarning: Argument 'interpolation' of type int is deprecated since 0.13 and will be removed in 0.15. Please use InterpolationMode enum.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from modules.mplug import VideoCaptioning\n",
    "\n",
    "tool_vc = VideoCaptioning(\"cuda:7\")\n",
    "video_captioning = tool_vc.inference # input: f\"{video_path}\", output: caption (str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing BLIP2 for ImageCaptioning\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading checkpoint shards: 100%|██████████| 2/2 [00:39<00:00, 19.51s/it]\n"
     ]
    }
   ],
   "source": [
    "from modules.blip import ImageCaptioning\n",
    "\n",
    "tool_blip2 = ImageCaptioning(\"cuda:0\")\n",
    "frames_captioning = tool_blip2.inference # input: f\"{video_path}\", output: caption (str)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Video Processing (MoviePy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing MoviepyInterface\n"
     ]
    }
   ],
   "source": [
    "from modules.video_moviepy import MoviepyInterface\n",
    "\n",
    "movie_interface = MoviepyInterface()\n",
    "video_subclip = movie_interface.intercept_fragments # input: f\"{video_path}, {begin_second}, {end_second}\", output: new_video_path\n",
    "add_subtitles = movie_interface.add_subtitles # input: f\"{video_path}, {start_time}, {duration}, {instruct_text}\", output: new_video_path\n",
    "concat_videos = movie_interface.concat_videos # input: f\"{video_path1}, {video_path2}...\", output: new_video_path\n",
    "extract_audio = movie_interface.extract_audio # input: f\"{video_path}\", output: audio_path\n",
    "add_audio_to_video = movie_interface.add_audio_to_video # input: f\"{video_path}, {audio_path}\", output: new_video_path"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Video Generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.modelscope_t2v import ModelscopeT2V\n",
    "\n",
    "mst2v = ModelscopeT2V(\"cuda:1\")\n",
    "text2video = mst2v.inference # input: f\"{description}\", output: new_video_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing Video2Canny\n",
      "Initializing Video2Pose\n",
      "Initializing Video2Depth\n"
     ]
    }
   ],
   "source": [
    "from modules.annotator import Video2Canny, Video2Pose, Video2Depth\n",
    "\n",
    "tool_v2c = Video2Canny()\n",
    "tool_v2p = Video2Pose(\"cuda:1\")\n",
    "tool_v2d = Video2Depth(\"cuda:1\")\n",
    "\n",
    "video2canny = tool_v2c.inference # input: f\"{video_path}\", output: new_video_path\n",
    "video2pose = tool_v2p.inference # input: f\"{video_path}\", output: new_video_path\n",
    "video2depth = tool_v2d.inference # input: f\"{video_path}\", output: new_video_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You have disabled the safety checker for <class 'diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_controlnet.StableDiffusionControlNetPipeline'> by passing `safety_checker=None`. Ensure that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered results in services or applications open to the public. Both the diffusers team and Hugging Face strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling it only for use-cases that involve analyzing network behavior or auditing its results. For more information, please have a look at https://github.com/huggingface/diffusers/pull/254 .\n",
      "You have disabled the safety checker for <class 'diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_controlnet.StableDiffusionControlNetPipeline'> by passing `safety_checker=None`. Ensure that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered results in services or applications open to the public. Both the diffusers team and Hugging Face strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling it only for use-cases that involve analyzing network behavior or auditing its results. For more information, please have a look at https://github.com/huggingface/diffusers/pull/254 .\n",
      "You have disabled the safety checker for <class 'diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_controlnet.StableDiffusionControlNetPipeline'> by passing `safety_checker=None`. Ensure that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered results in services or applications open to the public. Both the diffusers team and Hugging Face strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling it only for use-cases that involve analyzing network behavior or auditing its results. For more information, please have a look at https://github.com/huggingface/diffusers/pull/254 .\n"
     ]
    }
   ],
   "source": [
    "from modules.text2video_zero import CannyText2Video, PoseText2Video, DepthText2Video, VideoPix2Pix\n",
    "\n",
    "pose_text2video = PoseText2Video(\"cuda:5\").inference  # input: f\"{video_path}, {prompt}\", output: new_video_path\n",
    "canny_text2video = CannyText2Video(\"cuda:5\").inference  # input: f\"{video_path}, {prompt}\", output: new_video_path\n",
    "depth_text2video = DepthText2Video(\"cuda:5\").inference  # input: f\"{video_path}, {prompt}\", output: new_video_path\n",
    "video_pix2pix = VideoPix2Pix(\"cuda:5\").inference # input: f\"{video_path}, {prompt}\", output: new_video_path"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Audio Generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "torch version does not support flash attention. You will get faster inference speed by upgrade torch to newest nightly version.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing Bark for Text2Audio\n",
      "Loading bark models for text2audio...\n"
     ]
    }
   ],
   "source": [
    "from modules.bark import Text2Audio\n",
    "\n",
    "tool_t2a = Text2Audio()\n",
    "text2audio = tool_t2a.text2audio   # input: f\"{text}\", output: audio_path\n",
    "text2music = tool_t2a.text2music   # input: f\"{text}\", output: audio_path"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiments\n",
    "\n",
    "The code snippet below requires modifying the parameters passed in by the function to your own input.\n",
    "\n",
    "For the specific input format, please refer to the notes above."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a video of a girl in a blue sweater in a mirror .\n"
     ]
    }
   ],
   "source": [
    "print(video_captioning(\"video/v0006.mp4\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Second 0: the woman is wearing a blue dress. Second 1: the girl is wearing a blue dress. Second 2: the woman is wearing a blue dress. Second 3: the girl is wearing a blue dress. Second 4: the girl is wearing a blue dress. Second 5: the girl is dancing. Second 6: the girl is wearing a blue dress. Second 7: the girl is wearing a blue dress. Second 8: the girl is wearing a blue dress. Second 9: the woman is dancing. Second 10: the girl is wearing a blue dress.\n"
     ]
    }
   ],
   "source": [
    "print(frames_captioning(\"video/v0006.mp4\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 25/25 [00:09<00:00,  2.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "./video/ee58c835.mp4\n"
     ]
    }
   ],
   "source": [
    "print(\"./\" + text2video(\"a goldendoodle playing in a park by a lake\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "./video/0ef7_edge_v0006_v0006.mp4\n"
     ]
    }
   ],
   "source": [
    "print(\"./\" + video2canny(\"video/v0006.mp4\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "./video/2748_pose_v0006_v0006.mp4\n"
     ]
    }
   ],
   "source": [
    "print(\"./\" + video2pose(\"video/v0006.mp4\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "./video/2c51_depth_v0006_v0006.mp4\n"
     ]
    }
   ],
   "source": [
    "print(\"./\" + video2depth(\"video/v0006.mp4\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing chunk 1 / 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:05<00:00,  3.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing chunk 2 / 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:04<00:00,  4.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing chunk 3 / 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:04<00:00,  4.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing chunk 4 / 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:04<00:00,  4.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing chunk 5 / 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:04<00:00,  4.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing chunk 6 / 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:04<00:00,  4.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing chunk 7 / 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:04<00:00,  4.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing chunk 8 / 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:04<00:00,  4.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing chunk 9 / 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:03<00:00,  5.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "./video/5d78_pose2video_0d50_v0006.mp4\n"
     ]
    }
   ],
   "source": [
    "print(\"./\" + pose_text2video(\"video/0d50_human-pose_1759_v0006.mp4, an astronaut dancing on the moon\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing chunk 1 / 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:11<00:00,  1.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing chunk 2 / 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:11<00:00,  1.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing chunk 3 / 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:04<00:00,  4.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "./video/227e_depth2video_7574_235ee6b2.mp4\n"
     ]
    }
   ],
   "source": [
    "print(\"./\" + depth_text2video(\"video/7574_depth_235ee6b2_235ee6b2.mp4, a tiger\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing chunk 1 / 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 50/50 [00:31<00:00,  1.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing chunk 2 / 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 50/50 [00:31<00:00,  1.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing chunk 3 / 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 50/50 [00:12<00:00,  3.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "./video/1bc4_pix2pix_235ee6b2_235ee6b2.mp4\n"
     ]
    }
   ],
   "source": [
    "print(\"./\" + video_pix2pix(\"video/235ee6b2.mp4, make it snowy\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:01<00:00, 66.13it/s]\n",
      "100%|██████████| 7/7 [00:04<00:00,  1.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./video/edafb26c.wav\n"
     ]
    }
   ],
   "source": [
    "print(\"./\" + text2audio(\"a goldendoodle playing in a park by a lake\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:02<00:00, 33.85it/s]\n",
      "100%|██████████| 13/13 [00:09<00:00,  1.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./video/0f17d609.wav\n"
     ]
    }
   ],
   "source": [
    "print(\"./\" + text2music(\"a goldendoodle playing in a park by a lake\"))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "chatvid",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}