media-libs/libaom: fix big-endian detection

* Backport big-endian patches
* Fix big-endian detection by passing CMake arg

The bug mentions there may still be some issues but it sounds like they're
with test data and not the tests/code itself.

Bug: https://bugs.chromium.org/p/aomedia/issues/detail?id=3487
Signed-off-by: Sam James <sam@gentoo.org>

4 files changed, 311 insertions, 1 deletions

diff --git a/media-libs/libaom/files/libaom-3.7.0-big-endian-tests.patch b/media-libs/libaom/files/libaom-3.7.0-big-endian-tests.patch
new file mode 100644
index 000000000000..38559c564726
--- /dev/null
+++ b/media-libs/libaom/files/libaom-3.7.0-big-endian-tests.patch
@@ -0,0 +1,43 @@
+https://bugs.chromium.org/p/aomedia/issues/detail?id=3487
+https://aomedia.googlesource.com/aom/+/7029529477e1473e6eb7417538cea18edc5e3bd0
+
+From 7029529477e1473e6eb7417538cea18edc5e3bd0 Mon Sep 17 00:00:00 2001
+From: Wan-Teh Chang <wtc@google.com>
+Date: Wed, 13 Sep 2023 10:55:05 -0700
+Subject: [PATCH] Fix big-endian bugs in CodingPathSync tests
+
+Change Serialize() to read uint16_t samples correctly. Although only the
+least significant byte of each sample is nonzero, we cannot assume the
+least significant byte of the c-th sample in `row` is row[c * 2]. That
+is correct only on little-endian systems.
+
+Bug: aomedia:3487
+Change-Id: I9919ce6e3c877608ca7488fe4cc6957bcfe8c4cc
+---
+ test/coding_path_sync.cc | 10 ++++++----
+ 1 file changed, 6 insertions(+), 4 deletions(-)
+
+diff --git a/test/coding_path_sync.cc b/test/coding_path_sync.cc
+index c3e51fd565..31bc2d56b0 100644
+--- a/test/coding_path_sync.cc
++++ b/test/coding_path_sync.cc
+@@ -130,11 +130,13 @@ std::vector<int16_t> Serialize(const aom_image_t *img) {
+ 
+     for (int r = 0; r < h; ++r) {
+       for (int c = 0; c < w; ++c) {
+-        unsigned char *row = img->planes[plane] + r * img->stride[plane];
+-        if (img->fmt & AOM_IMG_FMT_HIGHBITDEPTH)
+-          bytes.push_back(row[c * 2]);
+-        else
++        const unsigned char *row = img->planes[plane] + r * img->stride[plane];
++        if (img->fmt & AOM_IMG_FMT_HIGHBITDEPTH) {
++          const uint16_t *row16 = reinterpret_cast<const uint16_t *>(row);
++          bytes.push_back(row16[c]);
++        } else {
+           bytes.push_back(row[c]);
++        }
+       }
+     }
+   }
+-- 
+2.42.0
diff --git a/media-libs/libaom/files/libaom-3.7.0-big-endian.patch b/media-libs/libaom/files/libaom-3.7.0-big-endian.patch
new file mode 100644
index 000000000000..907e5fd1110e
--- /dev/null
+++ b/media-libs/libaom/files/libaom-3.7.0-big-endian.patch
@@ -0,0 +1,130 @@
+https://bugs.chromium.org/p/aomedia/issues/detail?id=3487
+https://aomedia.googlesource.com/aom/+/8b65d76f2f95b1bfdc82b8f135246cb367d444e6%5E%21/
+
+From 8b65d76f2f95b1bfdc82b8f135246cb367d444e6 Mon Sep 17 00:00:00 2001
+From: Wan-Teh Chang <wtc@google.com>
+Date: Tue, 12 Sep 2023 16:38:47 -0700
+Subject: [PATCH] Fix byte order issue in write_enc_data_to_out_buf
+
+Fix the byte order issue for big-endian systems introduced in
+https://aomedia-review.googlesource.com/c/aom/+/167581.
+
+Use the existing HToBE64 macro defined in aom_util/endian_inl.h and
+delete the new get_byteswap64 function from aom_ports/bitops.h.
+
+Based on the patch in bug aomedia:3487.
+
+Bug: aomedia:3487
+Change-Id: I56183e062a9f2b18c4c6158fd8e47c5062de85ee
+---
+ aom_dsp/entenc.h   |  7 ++++---
+ aom_ports/bitops.h | 35 +----------------------------------
+ 2 files changed, 5 insertions(+), 37 deletions(-)
+
+diff --git a/aom_dsp/entenc.h b/aom_dsp/entenc.h
+index 467e47bf56..d26f027ed0 100644
+--- a/aom_dsp/entenc.h
++++ b/aom_dsp/entenc.h
+@@ -13,7 +13,7 @@
+ #define AOM_AOM_DSP_ENTENC_H_
+ #include <stddef.h>
+ #include "aom_dsp/entcode.h"
+-#include "aom_ports/bitops.h"
++#include "aom_util/endian_inl.h"
+ 
+ #ifdef __cplusplus
+ extern "C" {
+@@ -87,13 +87,14 @@ static AOM_INLINE void propagate_carry_bwd(unsigned char *buf, uint32_t offs) {
+   } while (carry);
+ }
+ 
+-// Reverse byte order and write data to buffer adding the carry-bit
++// Convert to big-endian byte order and write data to buffer adding the
++// carry-bit
+ static AOM_INLINE void write_enc_data_to_out_buf(unsigned char *out,
+                                                  uint32_t offs, uint64_t output,
+                                                  uint64_t carry,
+                                                  uint32_t *enc_offs,
+                                                  uint8_t num_bytes_ready) {
+-  const uint64_t reg = get_byteswap64(output) >> ((8 - num_bytes_ready) << 3);
++  const uint64_t reg = HToBE64(output << ((8 - num_bytes_ready) << 3));
+   memcpy(&out[offs], &reg, 8);
+   // Propagate carry backwards if exists
+   if (carry) {
+diff --git a/aom_ports/bitops.h b/aom_ports/bitops.h
+index 3c5b992bde..7f4c165f5a 100644
+--- a/aom_ports/bitops.h
++++ b/aom_ports/bitops.h
+@@ -13,7 +13,6 @@
+ #define AOM_AOM_PORTS_BITOPS_H_
+ 
+ #include <assert.h>
+-#include <stdint.h>
+ 
+ #include "aom_ports/msvc.h"
+ #include "config/aom_config.h"
+@@ -34,12 +33,8 @@ extern "C" {
+ // These versions of get_msb() are only valid when n != 0 because all
+ // of the optimized versions are undefined when n == 0:
+ 
+-// get_byteswap64:
+-// Returns the number (uint64_t) with byte-positions reversed
+-// e.g. input 0x123456789ABCDEF0 returns 0xF0DEBC9A78563412
+-
+ // GCC compiler: https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html
+-// MSVC: https://learn.microsoft.com/en-us/cpp/c-runtime-library/
++// MSVC: https://learn.microsoft.com/en-us/cpp/intrinsics/compiler-intrinsics
+ 
+ // use GNU builtins where available.
+ #if defined(__GNUC__) && \
+@@ -48,10 +43,6 @@ static INLINE int get_msb(unsigned int n) {
+   assert(n != 0);
+   return 31 ^ __builtin_clz(n);
+ }
+-
+-static INLINE uint64_t get_byteswap64(uint64_t num) {
+-  return __builtin_bswap64(num);
+-}
+ #elif defined(USE_MSC_INTRINSICS)
+ #pragma intrinsic(_BitScanReverse)
+ 
+@@ -61,10 +52,6 @@ static INLINE int get_msb(unsigned int n) {
+   _BitScanReverse(&first_set_bit, n);
+   return first_set_bit;
+ }
+-
+-static INLINE uint64_t get_byteswap64(uint64_t num) {
+-  return _byteswap_uint64(num);
+-}
+ #undef USE_MSC_INTRINSICS
+ #else
+ static INLINE int get_msb(unsigned int n) {
+@@ -82,26 +69,6 @@ static INLINE int get_msb(unsigned int n) {
+   }
+   return log;
+ }
+-
+-static INLINE uint64_t get_byteswap64(uint64_t num) {
+-  uint64_t out = 0x00;
+-  uint64_t mask = 0xFF00000000000000;
+-  int bit_shift = 56;  // 7 bytes
+-  // 4 ms bytes
+-  do {
+-    out |= (num & mask) >> bit_shift;
+-    mask >>= 8;
+-    bit_shift -= 16;
+-  } while (bit_shift >= 0);
+-  // 4 ls bytes
+-  bit_shift = 8;  // 1 byte
+-  do {
+-    out |= (num & mask) << bit_shift;
+-    mask >>= 8;
+-    bit_shift += 16;
+-  } while (bit_shift <= 56);
+-  return out;
+-}
+ #endif
+ 
+ #ifdef __cplusplus
+-- 
+2.42.0
diff --git a/media-libs/libaom/libaom-3.7.0-r2.ebuild b/media-libs/libaom/libaom-3.7.0-r2.ebuild
new file mode 100644
index 000000000000..3fbef9beb31c
--- /dev/null
+++ b/media-libs/libaom/libaom-3.7.0-r2.ebuild
@@ -0,0 +1,133 @@
+# Copyright 1999-2023 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=8
+
+PYTHON_COMPAT=( python3_{10..11} )
+inherit cmake-multilib python-any-r1
+
+if [[ ${PV} == *9999* ]]; then
+	inherit git-r3
+	EGIT_REPO_URI="https://aomedia.googlesource.com/aom"
+else
+	# To update test data tarball, follow these steps:
+	# 1.  Clone the upstream repo and check out the relevant tag,
+	#	  or download the release tarball
+	# 2.  Regular cmake configure (options don't matter here):
+	#     cd build && cmake ..
+	# 3.  Set LIBAOM_TEST_DATA_PATH to the directory you want and
+	#     run the "make testdata" target:
+	#     LIBAOM_TEST_DATA_PATH=../libaom-3.7.0-testdata make testdata
+	#     This will download the test data from the internet.
+	# 4.  Create a tarball out of that directory.
+	#     cd .. && tar cvaf libaom-3.7.0-testdata.tar.xz libaom-3.7.0-testdata
+	SRC_URI="
+		https://storage.googleapis.com/aom-releases/${P}.tar.gz
+		test? ( https://dev.gentoo.org/~sam/distfiles/${CATEGORY}/${PN}/${P}-testdata.tar.xz )
+	"
+	KEYWORDS="~alpha ~amd64 ~arm ~arm64 ~hppa ~ia64 ~loong ~ppc ~ppc64 ~riscv ~sparc ~x86"
+fi
+
+DESCRIPTION="Alliance for Open Media AV1 Codec SDK"
+HOMEPAGE="https://aomedia.org https://aomedia.googlesource.com/aom/"
+
+LICENSE="BSD-2"
+SLOT="0/3"
+IUSE="big-endian doc +examples test"
+IUSE="${IUSE} cpu_flags_x86_mmx cpu_flags_x86_sse cpu_flags_x86_sse2 cpu_flags_x86_sse3 cpu_flags_x86_ssse3"
+IUSE="${IUSE} cpu_flags_x86_sse4_1 cpu_flags_x86_sse4_2 cpu_flags_x86_avx cpu_flags_x86_avx2"
+IUSE="${IUSE} cpu_flags_arm_neon"
+RESTRICT="!test? ( test )"
+
+REQUIRED_USE="
+	cpu_flags_x86_sse2? ( cpu_flags_x86_mmx )
+	cpu_flags_x86_ssse3? ( cpu_flags_x86_sse2 )
+"
+
+BDEPEND="${PYTHON_DEPS}
+	dev-lang/perl
+	abi_x86_32? ( dev-lang/yasm )
+	abi_x86_64? ( dev-lang/yasm )
+	abi_x86_x32? ( dev-lang/yasm )
+	doc? ( app-doc/doxygen )
+"
+
+# The PATENTS file is required to be distributed with this package, bug #682214
+DOCS=( PATENTS )
+
+PATCHES=(
+	"${FILESDIR}"/${PN}-3.4.0-posix-c-source-ftello.patch
+	"${FILESDIR}"/${PN}-3.7.0-allow-fortify-source.patch
+
+	"${FILESDIR}"/${P}-big-endian.patch
+	"${FILESDIR}"/${P}-big-endian-tests.patch
+)
+
+multilib_src_configure() {
+	local mycmakeargs=(
+		-DENABLE_CCACHE=OFF
+		-DENABLE_DOCS=$(multilib_native_usex doc ON OFF)
+		-DENABLE_EXAMPLES=$(multilib_native_usex examples ON OFF)
+		-DENABLE_NASM=OFF
+		-DENABLE_TESTS=$(usex test)
+		-DENABLE_TOOLS=ON
+		-DENABLE_WERROR=OFF
+		# https://bugs.chromium.org/p/aomedia/issues/detail?id=3487 shows
+		# that big endian detection doesn't really work otherwise.
+		-DCONFIG_BIG_ENDIAN=$(usex big-endian 1 0)
+
+		# Needs libjxl, currently unpackaged.
+		-DCONFIG_TUNE_BUTTERAUGLI=0
+
+		# neon support is assumed to be always enabled on arm64
+		-DENABLE_NEON=$(usex cpu_flags_arm_neon ON $(usex arm64 ON OFF))
+		# ENABLE_DSPR2 / ENABLE_MSA for mips
+		-DENABLE_MMX=$(usex cpu_flags_x86_mmx ON OFF)
+		-DENABLE_SSE=$(usex cpu_flags_x86_sse ON OFF)
+		-DENABLE_SSE2=$(usex cpu_flags_x86_sse2 ON OFF)
+		-DENABLE_SSE3=$(usex cpu_flags_x86_sse3 ON OFF)
+		-DENABLE_SSSE3=$(usex cpu_flags_x86_ssse3 ON OFF)
+		-DENABLE_SSE4_1=$(usex cpu_flags_x86_sse4_1 ON OFF)
+		-DENABLE_SSE4_2=$(usex cpu_flags_x86_sse4_2 ON OFF)
+		-DENABLE_AVX=$(usex cpu_flags_x86_avx ON OFF)
+		-DENABLE_AVX2=$(usex cpu_flags_x86_avx2 ON OFF)
+	)
+
+	# For 32-bit multilib builds, force some intrinsics on to work around
+	# bug #816027. libaom seems to do runtime detection for some targets
+	# at least, so this isn't an issue.
+	if ! multilib_is_native_abi && use amd64 ; then
+		mycmakeargs+=(
+			-DENABLE_SSE3=ON
+			-DENABLE_SSSE3=ON
+		)
+	fi
+
+	# On ARM32-on-ARM64, things end up failing if NEON is off, bug #835456
+	# Just force generic, given it's a niche situation.
+	# TODO: could try forcing armv7 or similar?
+	if use arm && ! use cpu_flags_arm_neon && [[ $(uname -p) == "aarch64" ]] ; then
+		ewarn "Forcing generic for arm32-on-arm64 build (bug #835456)"
+		mycmakeargs+=(
+			-DAOM_TARGET_CPU=generic
+		)
+	fi
+
+	cmake_src_configure
+}
+
+multilib_src_test() {
+	LIBAOM_TEST_DATA_PATH="${WORKDIR}/${P}-testdata" "${BUILD_DIR}"/test_libaom || die
+}
+
+multilib_src_install() {
+	if multilib_is_native_abi && use doc ; then
+		local HTML_DOCS=( "${BUILD_DIR}"/docs/html/. )
+	fi
+
+	cmake_src_install
+}
+
+multilib_src_install_all() {
+	find "${ED}" -type f \( -name "*.a" -o -name "*.la" \) -delete || die
+}
diff --git a/media-libs/libaom/libaom-9999.ebuild b/media-libs/libaom/libaom-9999.ebuild
index 76a2b4d37365..80b545dbde32 100644
--- a/media-libs/libaom/libaom-9999.ebuild
+++ b/media-libs/libaom/libaom-9999.ebuild
@@ -33,7 +33,7 @@ HOMEPAGE="https://aomedia.org https://aomedia.googlesource.com/aom/"
 
 LICENSE="BSD-2"
 SLOT="0/3"
-IUSE="doc +examples test"
+IUSE="big-endian doc +examples test"
 IUSE="${IUSE} cpu_flags_x86_mmx cpu_flags_x86_sse cpu_flags_x86_sse2 cpu_flags_x86_sse3 cpu_flags_x86_ssse3"
 IUSE="${IUSE} cpu_flags_x86_sse4_1 cpu_flags_x86_sse4_2 cpu_flags_x86_avx cpu_flags_x86_avx2"
 IUSE="${IUSE} cpu_flags_arm_neon"
@@ -57,6 +57,7 @@ DOCS=( PATENTS )
 
 PATCHES=(
 	"${FILESDIR}"/${PN}-3.4.0-posix-c-source-ftello.patch
+	"${FILESDIR}"/${PN}-3.7.0-allow-fortify-source.patch
 )
 
 multilib_src_configure() {
@@ -68,6 +69,9 @@ multilib_src_configure() {
 		-DENABLE_TESTS=$(usex test)
 		-DENABLE_TOOLS=ON
 		-DENABLE_WERROR=OFF
+		# https://bugs.chromium.org/p/aomedia/issues/detail?id=3487 shows
+		# that big endian detection doesn't really work otherwise.
+		-DCONFIG_BIG_ENDIAN=$(usex big-endian 1 0)
 
 		# Needs libjxl, currently unpackaged.
 		-DCONFIG_TUNE_BUTTERAUGLI=0