From: Tobias Geerinckx-Rice <me@tobias.gr>
Date: Sun Sep 24 02:00:00 2023 +0200
Subject: libvpx: Fix CVE-2023-5217.

These are the changes made to libvpx between Firefox ESR 115.3.0 and
115.3.1, which claims to fix CVE-2023-5217 and so do we.  The report
itself is not public.

It consists of the following 2 upstream libvpx commits:
  af6dedd715f4307669366944cca6e0417b290282
  3fbd1dca6a4d2dad332a2110d646e4ffef36d590
which are not yet part of an upstream commit.

---
diff -Naur libvpx.orig/test/encode_api_test.cc libvpx/test/encode_api_test.cc
--- libvpx.orig/test/encode_api_test.cc	1970-01-01 01:00:01.000000000 +0100
+++ libvpx/test/encode_api_test.cc	2023-09-29 21:00:03.189620452 +0200
@@ -304,7 +304,6 @@
 
 void InitCodec(const vpx_codec_iface_t &iface, int width, int height,
                vpx_codec_ctx_t *enc, vpx_codec_enc_cfg_t *cfg) {
-  ASSERT_EQ(vpx_codec_enc_config_default(&iface, cfg, 0), VPX_CODEC_OK);
   cfg->g_w = width;
   cfg->g_h = height;
   cfg->g_lag_in_frames = 0;
@@ -342,6 +341,7 @@
         vpx_codec_ctx_t ctx = {};
       } enc;
 
+      ASSERT_EQ(vpx_codec_enc_config_default(iface, &cfg, 0), VPX_CODEC_OK);
       EXPECT_NO_FATAL_FAILURE(
           InitCodec(*iface, kWidth, kHeight, &enc.ctx, &cfg));
       if (IsVP9(iface)) {
@@ -353,6 +353,50 @@
 
       for (const auto threads : { 1, 4, 8, 6, 2, 1 }) {
         cfg.g_threads = threads;
+        EXPECT_NO_FATAL_FAILURE(EncodeWithConfig(cfg, &enc.ctx))
+            << "iteration: " << i << " threads: " << threads;
+      }
+    }
+  }
+}
+
+TEST(EncodeAPI, ConfigResizeChangeThreadCount) {
+  constexpr int kInitWidth = 1024;
+  constexpr int kInitHeight = 1024;
+
+  for (const auto *iface : kCodecIfaces) {
+    SCOPED_TRACE(vpx_codec_iface_name(iface));
+    for (int i = 0; i < (IsVP9(iface) ? 2 : 1); ++i) {
+      vpx_codec_enc_cfg_t cfg = {};
+      struct Encoder {
+        ~Encoder() { EXPECT_EQ(vpx_codec_destroy(&ctx), VPX_CODEC_OK); }
+        vpx_codec_ctx_t ctx = {};
+      } enc;
+
+      ASSERT_EQ(vpx_codec_enc_config_default(iface, &cfg, 0), VPX_CODEC_OK);
+      // Start in threaded mode to ensure resolution and thread related
+      // allocations are updated correctly across changes in resolution and
+      // thread counts. See https://crbug.com/1486441.
+      cfg.g_threads = 4;
+      EXPECT_NO_FATAL_FAILURE(
+          InitCodec(*iface, kInitWidth, kInitHeight, &enc.ctx, &cfg));
+      if (IsVP9(iface)) {
+        EXPECT_EQ(vpx_codec_control_(&enc.ctx, VP9E_SET_TILE_COLUMNS, 6),
+                  VPX_CODEC_OK);
+        EXPECT_EQ(vpx_codec_control_(&enc.ctx, VP9E_SET_ROW_MT, i),
+                  VPX_CODEC_OK);
+      }
+
+      cfg.g_w = 1000;
+      cfg.g_h = 608;
+      EXPECT_EQ(vpx_codec_enc_config_set(&enc.ctx, &cfg), VPX_CODEC_OK)
+          << vpx_codec_error_detail(&enc.ctx);
+
+      cfg.g_w = 16;
+      cfg.g_h = 720;
+
+      for (const auto threads : { 1, 4, 8, 6, 2, 1 }) {
+        cfg.g_threads = threads;
         EXPECT_NO_FATAL_FAILURE(EncodeWithConfig(cfg, &enc.ctx))
             << "iteration: " << i << " threads: " << threads;
       }
diff -Naur libvpx.orig/vp8/encoder/onyx_if.c libvpx/vp8/encoder/onyx_if.c
--- libvpx.orig/vp8/encoder/onyx_if.c	1970-01-01 01:00:01.000000000 +0100
+++ libvpx/vp8/encoder/onyx_if.c	2023-09-29 21:01:44.155476128 +0200
@@ -1443,6 +1443,11 @@
   last_h = cpi->oxcf.Height;
   prev_number_of_layers = cpi->oxcf.number_of_layers;
 
+  if (cpi->initial_width) {
+    // TODO(https://crbug.com/1486441): Allow changing thread counts; the
+    // allocation is done once in vp8_create_compressor().
+    oxcf->multi_threaded = cpi->oxcf.multi_threaded;
+  }
   cpi->oxcf = *oxcf;
 
   switch (cpi->oxcf.Mode) {