decode_gif: add transparency support and image offsets (#8419)

Author: NicolasHug

test/test_image.py

@@ -551,7 +551,9 @@ def test_pathlib_support(tmpdir):
  write_png(img, write_path)
 
 
-@pytest.mark.parametrize("name", ("gifgrid", "fire", "porsche", "treescap", "treescap-interlaced", "solid2", "x-trans"))
+@pytest.mark.parametrize(
+ "name", ("gifgrid", "fire", "porsche", "treescap", "treescap-interlaced", "solid2", "x-trans", "earth")
+)
 @pytest.mark.parametrize("scripted", (True, False))
 def test_decode_gif(tmpdir, name, scripted):
  # Using test images from GIFLIB
@@ -560,9 +562,16 @@ def test_decode_gif(tmpdir, name, scripted):
  # We're not testing against "welcome2" because PIL and GIFLIB disagee on what
  # the background color should be (likely a difference in the way they handle
  # transparency?)
+ # 'earth' image is from wikipedia, licensed under CC BY-SA 3.0
+ # https://creativecommons.org/licenses/by-sa/3.0/
+ # it allows to properly test for transparency, TOP-LEFT offsets, and
+ # disposal modes.
 
  path = tmpdir / f"{name}.gif"
- url = f"https://sourceforge.net/p/giflib/code/ci/master/tree/pic/{name}.gif?format=raw"
+ if name == "earth":
+ url = "https://upload.wikimedia.org/wikipedia/commons/2/2c/Rotating_earth_%28large%29.gif"
+ else:
+ url = f"https://sourceforge.net/p/giflib/code/ci/master/tree/pic/{name}.gif?format=raw"
  with open(path, "wb") as f:
  f.write(requests.get(url).content)
 

torchvision/csrc/io/image/cpu/decode_gif.cpp

@@ -86,32 +86,19 @@ torch::Tensor decode_gif(const torch::Tensor& encoded_data) {
  // This check should already done within DGifSlurp(), just to be safe
  TORCH_CHECK(num_images > 0, "GIF file should contain at least one image!");
 
- // Note:
- // The GIF format has this notion of "canvas" and "canvas size", where each
- // image could be displayed on the canvas at different offsets, forming a
- // mosaic/picture wall like so:
- //
- // <--- canvas W --->
- // ------------------------ ^
- // | | | |
- // | img1 | img3 | |
- // | |------------| canvas H
- // |---------- | |
- // | img2 | img4 | |
- // | | | |
- // ------------------------ v
- // The GifLib docs indicate that this is mostly vestigial
- // (https://giflib.sourceforge.net/whatsinagif/bits_and_bytes.html), and
- // modern viewers ignore the canvas size as well as image offsets. Hence,
- // we're ignoring that too:
- // - We're ignoring the canvas width and height and assume that the shape of
- // the canvas and of all images is the shape of the first image.
- // - We're enforcing that all images have the same shape.
- // - Left and Top offsets of each image are ignored as well and assumed to be
- // 0.
-
- auto out_h = gifFile->SavedImages[0].ImageDesc.Height;
- auto out_w = gifFile->SavedImages[0].ImageDesc.Width;
+ GifColorType bg = {0, 0, 0};
+ if (gifFile->SColorMap) {
+ bg = gifFile->SColorMap->Colors[gifFile->SBackGroundColor];
+ }
+
+ // The GIFLIB docs say that the canvas's height and width are potentially
+ // ignored by modern viewers, so to be on the safe side we set the output
+ // height to max(canvas_heigh, first_image_height). Same for width.
+ // https://giflib.sourceforge.net/whatsinagif/bits_and_bytes.html
+ auto out_h =
+ std::max(gifFile->SHeight, gifFile->SavedImages[0].ImageDesc.Height);
+ auto out_w =
+ std::max(gifFile->SWidth, gifFile->SavedImages[0].ImageDesc.Width);
 
  // We output a channels-last tensor for consistency with other image decoders.
  // Torchvision's resize tends to be is faster on uint8 channels-last tensors.
@@ -121,30 +108,59 @@ torch::Tensor decode_gif(const torch::Tensor& encoded_data) {
  auto out = torch::empty(
  {int64_t(num_images), 3, int64_t(out_h), int64_t(out_w)}, options);
  auto out_a = out.accessor<uint8_t, 4>();
-
  for (int i = 0; i < num_images; i++) {
  const SavedImage& img = gifFile->SavedImages[i];
- const GifImageDesc& desc = img.ImageDesc;
- TORCH_CHECK(
- desc.Width == out_w && desc.Height == out_h,
- "All images in the gif should have the same dimensions.");
 
+ GraphicsControlBlock gcb;
+ DGifSavedExtensionToGCB(gifFile, i, &gcb);
+
+ const GifImageDesc& desc = img.ImageDesc;
  const ColorMapObject* cmap =
  desc.ColorMap ? desc.ColorMap : gifFile->SColorMap;
  TORCH_CHECK(
  cmap != nullptr,
  "Global and local color maps are missing. This should never happen!");
 
+ // When going from one image to another, there is a "disposal method" which
+ // specifies how to handle the transition. E.g. DISPOSE_DO_NOT means that
+ // the current image should essentially be drawn on top of the previous
+ // canvas. The pixels of that previous canvas will appear on the new one if
+ // either:
+ // - a pixel is transparent in the current image
+ // - the current image is smaller than the canvas, hence exposing its pixels
+ // The "background" disposal method means that the current canvas should be
+ // set to the background color.
+ // We only support these 2 modes and default to "background" when the
+ // disposal method is unspecified, or when it's set to "DISPOSE_PREVIOUS"
+ // which according to GIFLIB is not widely supported.
+ // (https://giflib.sourceforge.net/whatsinagif/animation_and_transparency.html).
+ if (i > 0 && gcb.DisposalMode == DISPOSE_DO_NOT) {
+ out[i] = out[i - 1];
+ } else {
+ // Background. If bg wasn't defined, it will be (0, 0, 0)
+ for (int h = 0; h < gifFile->SHeight; h++) {
+ for (int w = 0; w < gifFile->SWidth; w++) {
+ out_a[i][0][h][w] = bg.Red;
+ out_a[i][1][h][w] = bg.Green;
+ out_a[i][2][h][w] = bg.Blue;
+ }
+ }
+ }
+
  for (int h = 0; h < desc.Height; h++) {
  for (int w = 0; w < desc.Width; w++) {
  auto c = img.RasterBits[h * desc.Width + w];
+ if (c == gcb.TransparentColor) {
+ continue;
+ }
  GifColorType rgb = cmap->Colors[c];
- out_a[i][0][h][w] = rgb.Red;
- out_a[i][1][h][w] = rgb.Green;
- out_a[i][2][h][w] = rgb.Blue;
+ out_a[i][0][h + desc.Top][w + desc.Left] = rgb.Red;
+ out_a[i][1][h + desc.Top][w + desc.Left] = rgb.Green;
+ out_a[i][2][h + desc.Top][w + desc.Left] = rgb.Blue;
  }
  }
  }
+
  out = out.squeeze(0); // remove batch dim if there's only one image
 
  DGifCloseFile(gifFile, &error);