buildroot/package/libsndfile/0009-nms_adpcm-fix-int-overflow-in-signal-estimate.patch

From 71565532463b22c24824101845a533a67bff4c9c Mon Sep 17 00:00:00 2001
From: Alex Stewart <alex.stewart@ni.com>
Date: Thu, 19 Oct 2023 14:07:19 -0400
Subject: [PATCH] nms_adpcm: fix int overflow in signal estimate

It is possible (though functionally incorrect) for the signal estimate
calculation in nms_adpcm_update() to overflow the int value of s_e,
resulting in undefined behavior.

Since adpcm state signal values are never practically larger than
16 bits, use smaller numeric sizes throughout the file to avoid the
overflow.

CVE: CVE-2022-33065
Fixes: https://github.com/libsndfile/libsndfile/issues/833

Authored-by: Arthur Taylor <art@ified.ca>
Signed-off-by: Alex Stewart <alex.stewart@ni.com>
Upstream: https://github.com/libsndfile/libsndfile/commit/71565532463b22c24824101845a533a67bff4c9c
[Peter: adjust for 1.2.2]
Signed-off-by: Peter Korsgaard <peter@korsgaard.com>
---
 src/nms_adpcm.c | 81 ++++++++++++++++++++++++-------------------------
 1 file changed, 40 insertions(+), 41 deletions(-)

diff --git a/src/nms_adpcm.c b/src/nms_adpcm.c
index 5999be1f..dca85f0b 100644
--- a/src/nms_adpcm.c
+++ b/src/nms_adpcm.c
@@ -48,36 +48,36 @@
 /* Variable names from ITU G.726 spec */
 struct nms_adpcm_state
 {	/* Log of the step size multiplier. Operated on by codewords. */
-	int yl ;
+	short yl ;
 
 	/* Quantizer step size multiplier. Generated from yl. */
-	int y ;
+	short y ;
 
 	/* Coefficents of the pole predictor */
-	int a [2] ;
+	short a [2] ;
 
 	/* Coefficents of the zero predictor  */
-	int b [6] ;
+	short b [6] ;
 
 	/* Previous quantized deltas (multiplied by 2^14) */
-	int d_q [7] ;
+	short d_q [7] ;
 
 	/* d_q [x] + s_ez [x], used by the pole-predictor for signs only. */
-	int p [3] ;
+	short p [3] ;
 
 	/* Previous reconstructed signal values. */
-	int s_r [2] ;
+	short s_r [2] ;
 
 	/* Zero predictor components of the signal estimate. */
-	int s_ez ;
+	short s_ez ;
 
 	/* Signal estimate, (including s_ez). */
-	int s_e ;
+	short s_e ;
 
 	/* The most recent codeword (enc:generated, dec:inputted) */
-	int Ik ;
+	char Ik ;
 
-	int parity ;
+	char parity ;
 
 	/*
 	** Offset into code tables for the bitrate.
@@ -109,7 +109,7 @@ typedef struct
 } NMS_ADPCM_PRIVATE ;
 
 /* Pre-computed exponential interval used in the antilog approximation. */
-static unsigned int table_expn [] =
+static unsigned short table_expn [] =
 {	0x4000, 0x4167, 0x42d5, 0x444c,	0x45cb, 0x4752, 0x48e2, 0x4a7a,
 	0x4c1b, 0x4dc7, 0x4f7a, 0x5138,	0x52ff, 0x54d1, 0x56ac, 0x5892,
 	0x5a82, 0x5c7e, 0x5e84, 0x6096,	0x62b4, 0x64dd, 0x6712, 0x6954,
@@ -117,21 +117,21 @@ static unsigned int table_expn [] =
 } ;
 
 /* Table mapping codewords to scale factor deltas. */
-static int table_scale_factor_step [] =
+static short table_scale_factor_step [] =
 {	0x0,	0x0,	0x0,	0x0,	0x4b0,	0x0,	0x0,	0x0,	/* 2-bit */
 	-0x3c,	0x0,	0x90,	0x0,	0x2ee,	0x0,	0x898,	0x0,	/* 3-bit */
 	-0x30,	0x12,	0x6b,	0xc8,	0x188,	0x2e0,	0x551,	0x1150,	/* 4-bit */
 } ;
 
 /* Table mapping codewords to quantized delta interval steps. */
-static unsigned int table_step [] =
+static unsigned short table_step [] =
 {	0x73F,	0,		0,		0,		0x1829,	0,		0,		0,		/* 2-bit */
 	0x3EB,	0,		0xC18,	0,		0x1581,	0,		0x226E,	0,		/* 3-bit */
 	0x20C,	0x635,	0xA83,	0xF12,	0x1418,	0x19E3,	0x211A,	0x2BBA,	/* 4-bit */
 } ;
 
 /* Binary search lookup table for quantizing using table_step. */
-static int table_step_search [] =
+static short table_step_search [] =
 {	0,		0x1F6D,	0,		-0x1F6D,	0,		0,			0,			0, /* 2-bit */
 	0x1008,	0x1192,	0,		-0x219A,	0x1656,	-0x1656,	0,			0, /* 3-bit */
 	0x872,	0x1277,	-0x8E6,	-0x232B,	0xD06,	-0x17D7,	-0x11D3,	0, /* 4-bit */
@@ -179,23 +179,23 @@ static sf_count_t nms_adpcm_seek (SF_PRIVATE *psf, int mode, sf_count_t offset)
 ** Maps [1,20480] to [1,1024] in an exponential relationship. This is
 ** approximately ret = b^exp where b = e^(ln(1024)/ln(20480)) ~= 1.0003385
 */
-static inline int
-nms_adpcm_antilog (int exp)
-{	int ret ;
+static inline short
+nms_adpcm_antilog (short exp)
+{	int_fast32_t r ;
 
-	ret = 0x1000 ;
-	ret += (((exp & 0x3f) * 0x166b) >> 12) ;
-	ret *= table_expn [(exp & 0x7c0) >> 6] ;
-	ret >>= (26 - (exp >> 11)) ;
+	r = 0x1000 ;
+	r += (((int_fast32_t) (exp & 0x3f) * 0x166b) >> 12) ;
+	r *= table_expn [(exp & 0x7c0) >> 6] ;
+	r >>= (26 - (exp >> 11)) ;
 
-	return ret ;
+	return (short) r ;
 } /* nms_adpcm_antilog */
 
 static void
 nms_adpcm_update (struct nms_adpcm_state *s)
 {	/* Variable names from ITU G.726 spec */
-	int a1ul ;
-	int fa1 ;
+	short a1ul, fa1 ;
+	int_fast32_t se ;
 	int i ;
 
 	/* Decay and Modify the scale factor in the log domain based on the codeword. */
@@ -222,7 +222,7 @@ nms_adpcm_update (struct nms_adpcm_state *s)
 	else if (fa1 > 256)
 		fa1 = 256 ;
 
-	s->a [0] = (0xff * s->a [0]) >> 8 ;
+	s->a [0] = (s->a [0] * 0xff) >> 8 ;
 	if (s->p [0] != 0 && s->p [1] != 0 && ((s->p [0] ^ s->p [1]) < 0))
 		s->a [0] -= 192 ;
 	else
@@ -230,7 +230,7 @@ nms_adpcm_update (struct nms_adpcm_state *s)
 		fa1 = -fa1 ;
 		}
 
-	s->a [1] = fa1 + ((0xfe * s->a [1]) >> 8) ;
+	s->a [1] = fa1 + ((s->a [1] * 0xfe) >> 8) ;
 	if (s->p [0] != 0 && s->p [2] != 0 && ((s->p [0] ^ s->p [2]) < 0))
 		s->a [1] -= 128 ;
 	else
@@ -250,19 +250,18 @@ nms_adpcm_update (struct nms_adpcm_state *s)
 			s->a [0] = a1ul ;
 		} ;
 
-	/* Compute the zero predictor estimate. Rotate past deltas too. */
-	s->s_ez = 0 ;
+	/* Compute the zero predictor estimate and rotate past deltas. */
+	se = 0 ;
 	for (i = 5 ; i >= 0 ; i--)
-	{	s->s_ez += s->d_q [i] * s->b [i] ;
+	{	se += (int_fast32_t) s->d_q [i] * s->b [i] ;
 		s->d_q [i + 1] = s->d_q [i] ;
 		} ;
+	s->s_ez = se >> 14 ;
 
-	/* Compute the signal estimate. */
-	s->s_e = s->a [0] * s->s_r [0] + s->a [1] * s->s_r [1] + s->s_ez ;
-
-	/* Return to scale */
-	s->s_ez >>= 14 ;
-	s->s_e >>= 14 ;
+	/* Complete the signal estimate. */
+	se += (int_fast32_t) s->a [0] * s->s_r [0] ;
+	se += (int_fast32_t) s->a [1] * s->s_r [1] ;
+	s->s_e = se >> 14 ;
 
 	/* Rotate members to prepare for next iteration. */
 	s->s_r [1] = s->s_r [0] ;
@@ -274,7 +273,7 @@ nms_adpcm_update (struct nms_adpcm_state *s)
 static int16_t
 nms_adpcm_reconstruct_sample (struct nms_adpcm_state *s, uint8_t I)
 {	/* Variable names from ITU G.726 spec */
-	int dqx ;
+	int_fast32_t dqx ;
 
 	/*
 	** The ordering of the 12-bit right-shift is a precision loss. It agrees
@@ -308,17 +307,17 @@ nms_adpcm_codec_init (struct nms_adpcm_state *s, enum nms_enc_type type)
 /*
 ** nms_adpcm_encode_sample()
 **
-** Encode a linear 16-bit pcm sample into a 2,3, or 4 bit NMS-ADPCM codeword
+** Encode a linear 16-bit pcm sample into a 2, 3, or 4 bit NMS-ADPCM codeword
 ** using and updating the predictor state.
 */
 static uint8_t
 nms_adpcm_encode_sample (struct nms_adpcm_state *s, int16_t sl)
 {	/* Variable names from ITU G.726 spec */
-	int d ;
+	int_fast32_t d ;
 	uint8_t I ;
 
 	/* Down scale the sample from 16 => ~14 bits. */
-	sl = (sl * 0x1fdf) / 0x7fff ;
+	sl = ((int_fast32_t) sl * 0x1fdf) / 0x7fff ;
 
 	/* Compute estimate, and delta from actual value */
 	nms_adpcm_update (s) ;
@@ -407,7 +406,7 @@ nms_adpcm_encode_sample (struct nms_adpcm_state *s, int16_t sl)
 */
 static int16_t
 nms_adpcm_decode_sample (struct nms_adpcm_state *s, uint8_t I)
-{	int sl ;
+{	int_fast32_t sl ;
 
 	nms_adpcm_update (s) ;
 	sl = nms_adpcm_reconstruct_sample (s, I) ;
-- 
2.39.5