Skip to content

Reduce memory usage peak when using thumbnail animated WebP decoding and encoding #72

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

Reduce memory usage peak when using thumbnail animated WebP decoding and encoding #72

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9b72246
Test to support thumbnail animated webp without using CGImage re-scal…
dreampiggy Mar 8, 2023
6af2dcf
Support using CoreGraphics to produce scaled down CGImage directly
dreampiggy Mar 8, 2023
File filter

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Support using CoreGraphics to produce scaled down CGImage directly
  • Loading branch information
@dreampiggy
dreampiggy committed Mar 8, 2023
commit 6af2dcfbcd69b21e18a3fb590735f4c5106e50be
8 changes: 4 additions & 4 deletions Example/SDWebImageWebPCoderExample/ViewController.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,10 @@ - (void)viewDidLoad {
NSURL *staticWebPURL = [NSURL URLWithString:@"http://littlesvr.ca/apng/images/SteamEngine.webp"];
NSURL *animatedWebPURL = [NSURL URLWithString:@"http://littlesvr.ca/apng/images/SteamEngine.webp?foo=bar"];

[self.imageView1 sd_setImageWithURL:staticWebPURL placeholderImage:nil options:0 context:@{SDWebImageContextImageThumbnailPixelSize : @(CGSizeMake(300, 300))} progress:nil completed:^(UIImage * _Nullable image, NSError * _Nullable error, SDImageCacheType cacheType, NSURL * _Nullable imageURL) {
}];
NSDictionary *decodeOptions = @{SDWebImageContextImageThumbnailPixelSize : @(CGSizeMake(300, 300)), SDImageCoderDecodeUseLazyDecoding: @(YES)}; // <---
[self.imageView2 sd_setImageWithURL:animatedWebPURL placeholderImage:nil options:0 context:@{SDWebImageContextImageDecodeOptions : decodeOptions} progress:nil completed:^(UIImage * _Nullable image, NSError * _Nullable error, SDImageCacheType cacheType, NSURL * _Nullable imageURL) {
// [self.imageView1 sd_setImageWithURL:staticWebPURL placeholderImage:nil options:0 context:@{SDWebImageContextImageThumbnailPixelSize : @(CGSizeMake(300, 300))} progress:nil completed:^(UIImage * _Nullable image, NSError * _Nullable error, SDImageCacheType cacheType, NSURL * _Nullable imageURL) {
// }];
NSDictionary *decodeOptions = @{SDImageCoderDecodeThumbnailPixelSize : @(CGSizeMake(300, 300)), SDImageCoderDecodeUseLazyDecoding: @(YES)}; // <---
[self.imageView2 sd_setImageWithURL:animatedWebPURL placeholderImage:nil options:0 context:@{SDWebImageContextImageDecodeOptions : decodeOptions, SDWebImageContextImageThumbnailPixelSize : @(CGSizeMake(300, 300))} progress:nil completed:^(UIImage * _Nullable image, NSError * _Nullable error, SDImageCacheType cacheType, NSURL * _Nullable imageURL) {
}];
}

Expand Down
72 changes: 30 additions & 42 deletions SDWebImageWebPCoder/Classes/SDImageWebPCoder.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,14 @@
#endif
#endif

static inline CGImageRef __nullable CGBitmapContextCreateScaledImage(cg_nullable CGContextRef canvas, CGSize scaledSize) CF_RETURNS_RETAINED {
if (!canvas) return NULL;
CGContextSaveGState(canvas);
CGContextScaleCTM(canvas, scaledSize.width, scaledSize.height);
CGContextRestoreGState(canvas);
return CGBitmapContextCreateImage(canvas);
}

@interface SDWebPCoderFrame : NSObject

@property (nonatomic, assign) NSUInteger index; // Frame index (zero based)
Expand Down Expand Up @@ -106,7 +114,6 @@ @implementation SDImageWebPCoder {
NSUInteger _currentBlendIndex;
BOOL _preserveAspectRatio;
CGSize _thumbnailSize;
BOOL _lazyDecode;
}

- (void)dealloc {
Expand Down Expand Up @@ -233,7 +240,7 @@ - (UIImage *)decodedImageWithData:(NSData *)data options:(nullable SDImageCoderO

do {
@autoreleasepool {
CGImageRef imageRef = [self sd_drawnWebpImageWithCanvas:canvas iterator:iter colorSpace:colorSpace scaledSize:scaledSize lazyDecode:NO];
CGImageRef imageRef = [self sd_drawnWebpImageWithCanvas:canvas iterator:iter colorSpace:colorSpace scaledSize:scaledSize];
if (!imageRef) {
continue;
}
Expand Down Expand Up @@ -390,7 +397,14 @@ - (UIImage *)incrementalDecodedImageWithOptions:(SDImageCoderOptions *)options {

// Only draw the last_y image height, keep remains transparent, in Core Graphics coordinate system
CGContextDrawImage(canvas, CGRectMake(0, height - last_y, width, last_y), imageRef);
CGImageRef newImageRef = CGBitmapContextCreateImage(canvas);
// Check whether we need to use thumbnail
CGImageRef newImageRef;
CGSize scaledSize = [SDImageCoderHelper scaledSizeWithImageSize:CGSizeMake(width, height) scaleSize:_thumbnailSize preserveAspectRatio:_preserveAspectRatio shouldScaleUp:NO];
if (!CGSizeEqualToSize(CGSizeMake(width, height), scaledSize)) {
newImageRef = CGBitmapContextCreateScaledImage(canvas, scaledSize);
} else {
newImageRef = CGBitmapContextCreateImage(canvas);
}
CGImageRelease(imageRef);
if (!newImageRef) {
CGContextRelease(canvas);
Expand All @@ -404,13 +418,6 @@ - (UIImage *)incrementalDecodedImageWithOptions:(SDImageCoderOptions *)options {
scale = 1;
}
}
CGSize scaledSize = [SDImageCoderHelper scaledSizeWithImageSize:CGSizeMake(width, height) scaleSize:_thumbnailSize preserveAspectRatio:_preserveAspectRatio shouldScaleUp:NO];
// Check whether we need to use thumbnail
if (!CGSizeEqualToSize(CGSizeMake(width, height), scaledSize)) {
CGImageRef scaledImageRef = [SDImageCoderHelper CGImageCreateScaled:newImageRef size:scaledSize];
CGImageRelease(newImageRef);
newImageRef = scaledImageRef;
}

#if SD_UIKIT || SD_WATCH
image = [[UIImage alloc] initWithCGImage:newImageRef scale:scale orientation:UIImageOrientationUp];
Expand Down Expand Up @@ -449,14 +456,8 @@ - (void)sd_blendWebpImageWithCanvas:(CGContextRef)canvas iterator:(WebPIterator)
}
}

- (nullable CGImageRef)sd_drawnWebpImageWithCanvas:(CGContextRef)canvas iterator:(WebPIterator)iter colorSpace:(nonnull CGColorSpaceRef)colorSpaceRef scaledSize:(CGSize)scaledSize lazyDecode:(BOOL)lazyDecode CF_RETURNS_RETAINED {
CGSize decodeSize;
if (!lazyDecode) {
decodeSize = CGSizeZero;
} else {
decodeSize = scaledSize;
}
CGImageRef imageRef = [self sd_createWebpImageWithData:iter.fragment colorSpace:colorSpaceRef scaledSize:decodeSize];
- (nullable CGImageRef)sd_drawnWebpImageWithCanvas:(CGContextRef)canvas iterator:(WebPIterator)iter colorSpace:(nonnull CGColorSpaceRef)colorSpaceRef scaledSize:(CGSize)scaledSize CF_RETURNS_RETAINED {
CGImageRef imageRef = [self sd_createWebpImageWithData:iter.fragment colorSpace:colorSpaceRef scaledSize:CGSizeZero];
if (!imageRef) {
return nil;
}
Expand All @@ -474,28 +475,22 @@ - (nullable CGImageRef)sd_drawnWebpImageWithCanvas:(CGContextRef)canvas iterator
CGContextClearRect(canvas, imageRect);
}
CGContextDrawImage(canvas, imageRect, imageRef);
CGImageRef newImageRef = CGBitmapContextCreateImage(canvas);

CGImageRef newImageRef;
// Check whether we need to use thumbnail
if (!CGSizeEqualToSize(CGSizeMake(canvasWidth, canvasHeight), scaledSize)) {
// Use CoreGraphics canvas to scale down, no need extra allocation
newImageRef = CGBitmapContextCreateScaledImage(canvas, scaledSize);
} else {
newImageRef = CGBitmapContextCreateImage(canvas);
}

CGImageRelease(imageRef);

if (iter.dispose_method == WEBP_MUX_DISPOSE_BACKGROUND) {
CGContextClearRect(canvas, imageRect);
}

if (!lazyDecode) {
// Check whether we need to use thumbnail
if (!CGSizeEqualToSize(CGSizeMake(canvasWidth, canvasHeight), scaledSize)) {
// Important: For Animated WebP thumbnail generation, we can not just use a scaled small canvas and draw each thumbnail frame
// This works **On Theory**. However, image scale down loss details. Animated WebP use the partial pixels with blend mode / dispose method with offset, to cover previous canvas status
// Because of this reason, even each frame contains small zigzag, the final animation contains visible glitch, this is not we want.
// So, always create the full pixels canvas (even though this consume more RAM), after drawn on the canvas, re-scale again with the final size
CGImageRef scaledImageRef = [SDImageCoderHelper CGImageCreateScaled:newImageRef size:scaledSize];
CGImageRelease(newImageRef);
newImageRef = scaledImageRef;
}

}

return newImageRef;
}

Expand All @@ -520,7 +515,6 @@ - (nullable CGImageRef)sd_createWebpImageWithData:(WebPData)webpData colorSpace:
// Use scaling for thumbnail
if (scaledSize.width != 0 && scaledSize.height != 0) {
config.options.use_scaling = 1;
config.options.no_fancy_upsampling = 1;
config.options.scaled_width = scaledSize.width;
config.options.scaled_height = scaledSize.height;
}
Expand Down Expand Up @@ -977,12 +971,6 @@ - (instancetype)initWithAnimatedImageData:(NSData *)data options:(nullable SDIma
preserveAspectRatio = preserveAspectRatioValue.boolValue;
}
_preserveAspectRatio = preserveAspectRatio;
BOOL lazyDecode = NO; // Defaults NO for animated image coder
NSNumber *lazyDecodeValue = options[SDImageCoderDecodeUseLazyDecoding];
if (lazyDecodeValue != nil) {
lazyDecode = lazyDecodeValue.boolValue;
}
_lazyDecode = lazyDecode;
_scale = scale;
_demux = demuxer;
_imageData = data;
Expand Down Expand Up @@ -1163,7 +1151,7 @@ - (UIImage *)safeStaticImageFrame {
_canvas = canvas;
}
CGSize scaledSize = [SDImageCoderHelper scaledSizeWithImageSize:CGSizeMake(_canvasWidth, _canvasHeight) scaleSize:_thumbnailSize preserveAspectRatio:_preserveAspectRatio shouldScaleUp:NO];
imageRef = [self sd_drawnWebpImageWithCanvas:_canvas iterator:iter colorSpace:_colorSpace scaledSize:scaledSize lazyDecode:NO];
imageRef = [self sd_drawnWebpImageWithCanvas:_canvas iterator:iter colorSpace:_colorSpace scaledSize:scaledSize];
} else {
CGSize scaledSize = [SDImageCoderHelper scaledSizeWithImageSize:CGSizeMake(iter.width, iter.height) scaleSize:_thumbnailSize preserveAspectRatio:_preserveAspectRatio shouldScaleUp:NO];
imageRef = [self sd_createWebpImageWithData:iter.fragment colorSpace:_colorSpace scaledSize:scaledSize];
Expand Down Expand Up @@ -1244,7 +1232,7 @@ - (UIImage *)safeAnimatedImageFrameAtIndex:(NSUInteger)index {
// Now the canvas is ready, which respects of dispose method behavior. Just do normal decoding and produce image.
// Check whether we need to use thumbnail
CGSize scaledSize = [SDImageCoderHelper scaledSizeWithImageSize:CGSizeMake(_canvasWidth, _canvasHeight) scaleSize:_thumbnailSize preserveAspectRatio:_preserveAspectRatio shouldScaleUp:NO];
CGImageRef imageRef = [self sd_drawnWebpImageWithCanvas:_canvas iterator:iter colorSpace:_colorSpace scaledSize:scaledSize lazyDecode:_lazyDecode];
CGImageRef imageRef = [self sd_drawnWebpImageWithCanvas:_canvas iterator:iter colorSpace:_colorSpace scaledSize:scaledSize];
if (!imageRef) {
return nil;
}
Expand Down