fix potential memory leaks for failing mixing scaling on very large images

diff --git a/Changes b/Changes
index b153ca02f0a90e394e2581ee3fc1e3eb5d319fc1..f6eb87be23e46f5d58e9726b4e95c8b0237ed579 100644 (file)
--- a/Changes
+++ b/Changes
@@ -5,6 +5,11 @@ Coverity finally finished a build, fix a few problems:
  - reading a color-mapped TGA file with an id string would cause a
    double-free if the palette was truncated.
 
+ - mixing scaling to sizes where the accumulator row, the working
+   output row or the working input row didn't fit into the address
+   space could result in memory leaks.  This can probably only be
+   reproduced on very wide floating-point sample images.
+
 Imager 1.008 - 31 Dec 2018
 ============
 

diff --git a/scale.im b/scale.im
index 4e45a3006c03991ab267ff8cdcc3f805b1b60488..5ed637b977c0aeab5c9068067801724dd75e2a73 100644 (file)
--- a/scale.im
+++ b/scale.im
@@ -50,7 +50,7 @@ Adapted from pnmscale.
 */
 i_img *
 i_scale_mixing(i_img *src, i_img_dim x_out, i_img_dim y_out) {
-  i_img *result;
+  i_img *result = NULL;
   i_fcolor *accum_row = NULL;
   i_img_dim x, y;
   int ch;
@@ -79,16 +79,16 @@ i_scale_mixing(i_img *src, i_img_dim x_out, i_img_dim y_out) {
 
   y_scale = y_out / (double)src->ysize;
 
-  result = i_sametype_chans(src, x_out, y_out, src->channels);
-  if (!result)
-    return NULL;
-
   accum_row_bytes = sizeof(i_fcolor) * src->xsize;
   if (accum_row_bytes / sizeof(i_fcolor) != src->xsize) {
     i_push_error(0, "integer overflow allocating accumulator row buffer");
     return NULL;
   }
 
+  result = i_sametype_chans(src, x_out, y_out, src->channels);
+  if (!result)
+    return NULL;
+
   accum_row  = mymalloc(accum_row_bytes);
 
 #code src->bits <= 8
@@ -98,11 +98,15 @@ i_scale_mixing(i_img *src, i_img_dim x_out, i_img_dim y_out) {
 
   in_row_bytes = sizeof(IM_COLOR) * src->xsize;
   if (in_row_bytes / sizeof(IM_COLOR) != src->xsize) {
+    myfree(accum_row);
+    i_img_destroy(result);
     i_push_error(0, "integer overflow allocating input row buffer");
     return NULL;
   }
   out_row_bytes = sizeof(IM_COLOR) * x_out;
   if (out_row_bytes / sizeof(IM_COLOR) != x_out) {
+    myfree(accum_row);
+    i_img_destroy(result);
     i_push_error(0, "integer overflow allocating output row buffer");
     return NULL;
   }