Bug 1784835. Use checkedint in webp encoder to avoid overflow. r=aosmond, a=RyanVM

image/encoders/webp/nsWebPEncoder.cpp

@@ -103,12 +103,20 @@ nsWebPEncoder::InitFromData(const uint8_t* aData,
 
   size_t size = 0;
 
+  CheckedInt32 width = CheckedInt32(aWidth);
+  CheckedInt32 height = CheckedInt32(aHeight);
+  CheckedInt32 stride = CheckedInt32(aStride);
+  if (!width.isValid() || !height.isValid() || !stride.isValid() ||
+      !(CheckedUint32(aStride) * CheckedUint32(aHeight)).isValid()) {
+    return NS_ERROR_INVALID_ARG;
+  }
+
   if (aInputFormat == INPUT_FORMAT_RGB) {
-    size =
-        WebPEncodeRGB(aData, aWidth, aHeight, aStride, quality, &mImageBuffer);
+    size = WebPEncodeRGB(aData, width.value(), height.value(), stride.value(),
+                         quality, &mImageBuffer);
   } else if (aInputFormat == INPUT_FORMAT_RGBA) {
-    size =
-        WebPEncodeRGBA(aData, aWidth, aHeight, aStride, quality, &mImageBuffer);
+    size = WebPEncodeRGBA(aData, width.value(), height.value(), stride.value(),
+                          quality, &mImageBuffer);
   } else if (aInputFormat == INPUT_FORMAT_HOSTARGB) {
     UniquePtr<uint8_t[]> aDest = MakeUnique<uint8_t[]>(aStride * aHeight);
 
@@ -135,8 +143,8 @@ nsWebPEncoder::InitFromData(const uint8_t* aData,
       }
     }
 
-    size = WebPEncodeRGBA(aDest.get(), aWidth, aHeight, aStride, quality,
-                          &mImageBuffer);
+    size = WebPEncodeRGBA(aDest.get(), width.value(), height.value(),
+                          stride.value(), quality, &mImageBuffer);
   }
 
   mFinished = true;