create pllava filter

DPF/filters/videos/pllava_filter.py

@@ -0,0 +1,177 @@
+import sys
+
+sys.path.append('pllava_filter_core/')
+sys.path.append('../../../')
+import os
+from io import BytesIO
+from typing import Any, Optional
+
+import numpy as np
+import torch
+import torchvision
+from decord import VideoReader, cpu
+from huggingface_hub import snapshot_download
+from PIL import Image
+from tasks.eval.eval_utils import conv_templates
+from tasks.eval.model_utils import load_pllava
+
+from DPF.filters.videos.video_filter import VideoFilter
+from DPF.types import ModalityToDataMapping
+
+
+def get_index(num_frames, num_segments):
+ seg_size = float(num_frames - 1) / num_segments
+ start = int(seg_size / 2)
+ offsets = np.array([
+ start + int(np.round(seg_size * idx)) for idx in range(num_segments)
+ ])
+ return offsets
+
+def load_video(video_path, num_segments=8, return_msg=False, num_frames=16, resolution=336):
+ transforms = torchvision.transforms.Resize(size=resolution)
+ vr = VideoReader(video_path, ctx=cpu(0), num_threads=1)
+ num_frames = len(vr)
+ frame_indices = get_index(num_frames, num_segments)
+ images_group = []
+ for frame_index in frame_indices:
+ img = Image.fromarray(vr[frame_index].asnumpy())
+ images_group.append(transforms(img))
+ if return_msg:
+ fps = float(vr.get_avg_fps())
+ sec = ", ".join([str(round(f / fps, 1)) for f in frame_indices])
+ # " " should be added in the start and end
+ msg = f"The video contains {len(frame_indices)} frames sampled at {sec} seconds."
+ return images_group, msg
+ else:
+ return images_group
+
+class PllavaFilter(VideoFilter):
+ """
+ Pllava inference class to get captions for videos.
+ More info about the model here: https://pllava.github.io
+ """
+ def __init__(
+ self,
+ model_path: str = 'ermu2001/pllava-34b',
+ weights_path: str = 'pllava_filter_core/MODELS/pllava-34b',
+ weights_dir: str = 'pllava_filter_core/MODELS/pllava-34b',
+ prompt: str = "short",
+ do_sample: bool = True,
+ batch_size: int = 16,
+ conv_mode: str = 'eval_vcg_llavanext',
+ device: str = "cuda:0",
+ workers: int = 16,
+ num_frames: int = 32,
+ max_new_tokens: int = 100,
+ num_segments: int = 32,
+ resolution: int = 672,
+ temperature: float = 0.1,
+ use_lora: bool = True,
+ lora_alpha: int = 4,
+ pbar: bool = True,
+ _pbar_position: int = 0,
+ use_multi_gpus: bool = True
+ ):
+ super().__init__(pbar, _pbar_position)
+ self.weights_dir = weights_dir
+ self.max_new_tokens = max_new_tokens
+ self.conv_mode = conv_mode
+ self.use_lora = use_lora
+ self.do_sample = do_sample
+ self.lora_alpha = lora_alpha
+ self.weights_path = weights_path
+ self.batch_size = batch_size
+ self.num_segments = batch_size
+ self.num_workers = workers
+ self.device = device
+ self.model_path = model_path
+ self.prompt_to_use = prompt
+ self.temperature = temperature
+ self.resolution = resolution
+ self.num_segments = num_segments
+ self.num_frames = num_frames
+ self.use_multi_gpus = use_multi_gpus
+ prompts = {
+ 'detailed_video': 'Please provide a caption for this image. Speak confidently and describe everything clearly. Do not lie and describe only what you can see',
+ 'pixart': 'Describe this image and its style in a very detailed manner',
+ 'short': 'Describe this image very shortly in 1-2 short sentences',
+ 'short-video': 'Describe this video very shortly in 1-2 short sentences. Describe what is happening in this video.'
+ }
+
+
+ if not os.path.exists(weights_path):
+
+ repo_ids = [
+ 'ermu2001/pllava-34b',
+ ]
+ for repo_id in repo_ids:
+ read_token = '...'
+ local_dir = repo_id.replace('ermu2001', 'pllava_filter_core/MODELS')
+ snapshot_download(
+ repo_id,
+ local_dir=local_dir,
+ repo_type='model',
+ local_dir_use_symlinks=True,
+ token=read_token,
+ )
+ self.model, self.processor = load_pllava(
+ self.weights_path,
+ self.num_frames,
+ use_lora=self.use_lora,
+ weight_dir=self.weights_dir,
+ lora_alpha=self.lora_alpha,
+ use_multi_gpus=True)
+
+ self.input_ids = prompts[self.prompt_to_use]
+
+ self.conv = conv_templates[self.conv_mode].copy()
+ self.conv.user_query(self.input_ids, is_mm=True)
+ self.prompt = self.conv.get_prompt()
+
+ @property
+ def result_columns(self) -> list[str]:
+ return [f"caption {self.model_path} prompt {self.prompt_to_use}"]
+
+ @property
+ def dataloader_kwargs(self) -> dict[str, Any]:
+ return {
+ "num_workers": self.num_workers,
+ "batch_size": self.batch_size,
+ "drop_last": False,
+ }
+
+ def preprocess_data(
+ self,
+ modality2data: ModalityToDataMapping,
+ metadata: dict[str, Any]
+ ) -> Any:
+ key = metadata[self.key_column]
+ video_file = BytesIO(modality2data['video'])
+ video_file, _ = load_video(video_file, num_segments=self.num_segments, return_msg=True, resolution=self.resolution)
+ return key, video_file
+
+ def process_batch(self, batch: list[Any]) -> dict[str, list[Any]]:
+ df_batch_labels = self._get_dict_from_schema()
+ keys, video_tensors = list(zip(*batch))
+ input_ids_batch = [self.prompt] * len(video_tensors)
+ inputs = self.processor(text=input_ids_batch, images=video_tensors, return_tensors="pt")
+ inputs = inputs.to(self.model.device)
+ with torch.no_grad():
+ output_token = self.model.generate(**inputs, media_type='video',
+ do_sample=self.do_sample, max_new_tokens=self.max_new_tokens,temperature=self.temperature
+ )
+ output_texts = self.processor.batch_decode(output_token, skip_special_tokens=True, clean_up_tokenization_spaces=True)
+ if self.conv.roles[-1] == "<|im_start|>assistant\n":
+ split_tag = "<|im_start|> assistant\n"
+ else:
+ split_tag = self.conv.roles[-1]
+ all_outputs: list[Optional[str]] = []
+ for output_text in output_texts:
+ output_text = output_text.split(split_tag)[-1]
+ ending = self.conv.sep if isinstance(self.conv.sep, str) else self.conv.sep[1]
+ output_text = output_text.removesuffix(ending).strip()
+ self.conv.messages[-1][1] = output_text
+ all_outputs.append(output_text)
+ df_batch_labels[self.schema[1]].extend(all_outputs)
+ df_batch_labels[self.key_column].extend(keys)
+ return df_batch_labels

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/.gitattributes

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/a18b9bd632b18f62b67b5c5043c9f8349765273a

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/README.md

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/8f7cb3626eb299ba8b03803ac0806aef0656142a

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/added_tokens.json

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/b81c25ab6a494a6ab8565da7c31707380341335f

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/config.json

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/dae2b5d99600ccf0222846887e067444074f09a8

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/generation_config.json

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/bb989af42f4048e75f8297cf9f82eb667d7e5ba7

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/model-00001-of-00015.safetensors

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/eb5984a52a1f0ba666a0bcde49f2e616903987e0ee6087e48e4318b9360bce88

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/model-00002-of-00015.safetensors

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/057c5f34030620ce17a4435b4bd42fb0c288fa212d0b871538f8a10a6d33795d

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/model-00003-of-00015.safetensors

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/efa83eae4b10e026329825b647d467f59a556de1f47a9ed5778beb2cf6f77f3e

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/model-00004-of-00015.safetensors

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/4941d3cdc160f395733e2f6095dbecfae9ec9f21e2593d38603164e328935900

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/model-00005-of-00015.safetensors

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/e484a8d26fb778d0f31e5973c82ef770b17ab62f13ab802fb6ff37cf37cf1d46

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/model-00006-of-00015.safetensors

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/7c5b848719cea638cc67b9de55983b910467b1d933e75acda84f8817b4d682e9

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/model-00007-of-00015.safetensors

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/dd9482fb59a916b280b5fbdbfe5665c4909970460a587f9d1a57380c94aafe5d

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/model-00008-of-00015.safetensors

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/53cf10aed2db791cb4737efaf079c62681b5f28c172ca4d6afb5767eebfbd329

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/model-00009-of-00015.safetensors

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/ea9e1607b1afc30907f82d5e2c772c11d37d35c557462449df8be49fd59b10db

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/model-00010-of-00015.safetensors

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/d65cfe302834b90fb10ed90eb08a4186a077a6c8fe12c58ccae1bbd4877ba7c9

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/model-00011-of-00015.safetensors

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/a491f2338e6404564a7dea4e8bff167fc8df4b2cee35e5fd354f1e76b2f7fc6e

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/model-00012-of-00015.safetensors

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/0cab9a7831dc11e7e0c41b3cf6cb7438056362b21364ec06b0239ca13bed4968

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/model-00013-of-00015.safetensors

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/452da7c5f2619ac19cffcf30a2121bd56fcea9f43a56e5a031f2e4dc0dc9c040

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/model-00014-of-00015.safetensors

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/e7f57097ed5d3ec2540f4b0878c4525b785859535047e73ab1cd0ca11ef224c1

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/model-00015-of-00015.safetensors

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/149724923406fa20d9c3488d9b746c9f869bf30e65fab2ccbc8b6c144e6d0b3c

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/model.safetensors-deprecated

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/7ff236d202deb53c3066e95bccfa256e617b658baca835298d20ded3186cfea0

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/model.safetensors.index.json

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/cc9d2062b37e81cbcd5e35252b8754444773bac5

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/preprocessor_config.json

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/c43fe86e999239f50fa9e2a16b905c1d46f5d7e8

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/processor_config.json

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/980f8df9c7fe43886b89519b0c117b355f814a61

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/special_tokens_map.json

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/955e1c435b490e2269213ffb8044faec38d7535d

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/tokenizer.json

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/1957d86c9429f01e08dac78da9371b2b2cc43f7b

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/tokenizer.model

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/386c49cf943d71aa110361135338c50e38beeff0a66593480421f37b319e1a39

DPF/filters/videos/pllava_filter_core/MODELS/pllava-34b/tokenizer_config.json

@@ -0,0 +1 @@
+../../../../../../../../.cache/huggingface/hub/models--ermu2001--pllava-34b/blobs/202082ec94c8de54770d4dc880a3a3218c434436

DPF/filters/videos/pllava_filter_core/models/pllava/__init__.py

@@ -0,0 +1,55 @@
+# Copyright 2023 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import TYPE_CHECKING
+
+from transformers.utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available
+
+
+_import_structure = {"configuration_pllava": ["PLLAVA_PRETRAINED_CONFIG_ARCHIVE_MAP", "PllavaConfig"]}
+
+try:
+ if not is_torch_available():
+ raise OptionalDependencyNotAvailable()
+except OptionalDependencyNotAvailable:
+ pass
+else:
+ _import_structure["modeling_pllava"] = [
+ "PLLAVA_PRETRAINED_MODEL_ARCHIVE_LIST",
+ "PllavaForConditionalGeneration",
+ "PllavaPreTrainedModel",
+ ]
+ _import_structure["processing_pllava"] = ["PllavaProcessor"]
+
+
+if TYPE_CHECKING:
+ from .configuration_pllava import PLLAVA_PRETRAINED_CONFIG_ARCHIVE_MAP, PllavaConfig
+
+ try:
+ if not is_torch_available():
+ raise OptionalDependencyNotAvailable()
+ except OptionalDependencyNotAvailable:
+ pass
+ else:
+ from .modeling_pllava import (
+ PLLAVA_PRETRAINED_MODEL_ARCHIVE_LIST,
+ PllavaForConditionalGeneration,
+ PllavaPreTrainedModel,
+ )
+ from .processing_pllava import PllavaProcessor
+
+
+else:
+ import sys
+
+ sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure)