media-video/rtmpdump: Fix OpenSSL 1.1, make libressl dep multilib

Fix issues with rtmpe streams and swf hash auto-calculation causing core dumps
See: https://github.com/xbmc/inputstream.rtmp/pull/46
Package-Manager: Portage-2.3.60, Repoman-2.3.12
Signed-off-by: Craig Andrews <candrews@gentoo.org>

diff --git a/media-video/rtmpdump/MBS_2018-12-07-13_16.flv b/media-video/rtmpdump/MBS_2018-12-07-13_16.flv
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/media-video/rtmpdump/OBC_2018-11-02-14_04.flv b/media-video/rtmpdump/OBC_2018-11-02-14_04.flv
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/media-video/rtmpdump/files/rtmpdump-openssl-1.1-v2.patch b/media-video/rtmpdump/files/rtmpdump-openssl-1.1-v2.patch
new file mode 100644 (file)
index 0000000..146243b
--- /dev/null
+++ b/media-video/rtmpdump/files/rtmpdump-openssl-1.1-v2.patch
@@ -0,0 +1,248 @@
+https://raw.githubusercontent.com/xbmc/inputstream.rtmp/master/depends/common/librtmp/0003-openssl-1.1.patch
+See also https://github.com/xbmc/inputstream.rtmp/pull/46
+--- a/librtmp/dh.h
++++ b/librtmp/dh.h
+@@ -253,20 +253,42 @@
+   if (!dh)
+     goto failed;
+ 
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+   MP_new(dh->g);
+ 
+   if (!dh->g)
+     goto failed;
++#else
++  BIGNUM *g = NULL;
++  MP_new(g);
++  if (!g)
++    goto failed;
++#endif
+ 
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+   MP_gethex(dh->p, P1024, res);       /* prime P1024, see dhgroups.h */
++#else
++  BIGNUM* p = NULL;
++  DH_get0_pqg(dh, (BIGNUM const**)&p, NULL, NULL);
++  MP_gethex(p, P1024, res); /* prime P1024, see dhgroups.h */
++#endif
+   if (!res)
+     {
+       goto failed;
+     }
+ 
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+   MP_set_w(dh->g, 2); /* base 2 */
++#else
++  MP_set_w(g, 2);   /* base 2 */
++  DH_set0_pqg(dh, p, NULL, g);
++#endif
+ 
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+   dh->length = nKeyBits;
++#else
++  DH_set_length(dh, nKeyBits);
++#endif
+   return dh;
+ 
+ failed:
+@@ -293,12 +315,24 @@
+       MP_gethex(q1, Q1024, res);
+       assert(res);
+ 
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+       res = isValidPublicKey(dh->pub_key, dh->p, q1);
++#else
++      BIGNUM const* pub_key = NULL;
++      BIGNUM const* p = NULL;
++      DH_get0_key(dh, &pub_key, NULL);
++      DH_get0_pqg(dh, &p, NULL, NULL);
++      res = isValidPublicKey((BIGNUM*)pub_key, (BIGNUM*)p, q1);
++#endif
+       if (!res)
+       {
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+         MP_free(dh->pub_key);
+         MP_free(dh->priv_key);
+         dh->pub_key = dh->priv_key = 0;
++#else
++          DH_free(dh);
++#endif
+       }
+ 
+       MP_free(q1);
+@@ -314,15 +348,29 @@
+ DHGetPublicKey(MDH *dh, uint8_t *pubkey, size_t nPubkeyLen)
+ {
+   int len;
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+   if (!dh || !dh->pub_key)
++#else
++  BIGNUM const* pub_key = NULL;
++  DH_get0_key(dh, &pub_key, NULL);
++  if (!dh || !pub_key)
++#endif
+     return 0;
+ 
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+   len = MP_bytes(dh->pub_key);
++#else
++  len = MP_bytes(pub_key);
++#endif
+   if (len <= 0 || len > (int) nPubkeyLen)
+     return 0;
+ 
+   memset(pubkey, 0, nPubkeyLen);
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+   MP_setbin(dh->pub_key, pubkey + (nPubkeyLen - len), len);
++#else
++  MP_setbin(pub_key, pubkey + (nPubkeyLen - len), len);
++#endif
+   return 1;
+ }
+ 
+@@ -364,7 +412,13 @@
+   MP_gethex(q1, Q1024, len);
+   assert(len);
+ 
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+   if (isValidPublicKey(pubkeyBn, dh->p, q1))
++#else
++  BIGNUM const* p = NULL;
++  DH_get0_pqg(dh, &p, NULL, NULL);
++  if (isValidPublicKey(pubkeyBn, (BIGNUM*)p, q1))
++#endif
+     res = MDH_compute_key(secret, nPubkeyLen, pubkeyBn, dh);
+   else
+     res = -1;
+--- a/librtmp/handshake.h
++++ b/librtmp/handshake.h
+@@ -31,9 +31,9 @@
+ #define SHA256_DIGEST_LENGTH  32
+ #endif
+ #define HMAC_CTX      sha2_context
+-#define HMAC_setup(ctx, key, len)     sha2_hmac_starts(&ctx, (unsigned char *)key, len, 0)
+-#define HMAC_crunch(ctx, buf, len)    sha2_hmac_update(&ctx, buf, len)
+-#define HMAC_finish(ctx, dig, dlen)   dlen = SHA256_DIGEST_LENGTH; sha2_hmac_finish(&ctx, dig)
++#define HMAC_setup(ctx, key, len)     sha2_hmac_starts(ctx, (unsigned char *)key, len, 0)
++#define HMAC_crunch(ctx, buf, len)    sha2_hmac_update(ctx, buf, len)
++#define HMAC_finish(ctx, dig, dlen)   dlen = SHA256_DIGEST_LENGTH; sha2_hmac_finish(ctx, dig)
+ 
+ typedef arc4_context *        RC4_handle;
+ #define RC4_alloc(h)  *h = malloc(sizeof(arc4_context))
+@@ -50,9 +50,9 @@
+ #endif
+ #undef HMAC_CTX
+ #define HMAC_CTX      struct hmac_sha256_ctx
+-#define HMAC_setup(ctx, key, len)     hmac_sha256_set_key(&ctx, len, key)
+-#define HMAC_crunch(ctx, buf, len)    hmac_sha256_update(&ctx, len, buf)
+-#define HMAC_finish(ctx, dig, dlen)   dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(&ctx, SHA256_DIGEST_LENGTH, dig)
++#define HMAC_setup(ctx, key, len)     hmac_sha256_set_key(ctx, len, key)
++#define HMAC_crunch(ctx, buf, len)    hmac_sha256_update(ctx, len, buf)
++#define HMAC_finish(ctx, dig, dlen)   dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(ctx, SHA256_DIGEST_LENGTH, dig)
+ #define HMAC_close(ctx)
+ 
+ typedef struct arcfour_ctx*   RC4_handle;
+@@ -64,14 +64,23 @@
+ 
+ #else /* USE_OPENSSL */
+ #include <openssl/sha.h>
++#include <openssl/ossl_typ.h>
+ #include <openssl/hmac.h>
+ #include <openssl/rc4.h>
+ #if OPENSSL_VERSION_NUMBER < 0x0090800 || !defined(SHA256_DIGEST_LENGTH)
+ #error Your OpenSSL is too old, need 0.9.8 or newer with SHA256
+ #endif
+-#define HMAC_setup(ctx, key, len)     HMAC_CTX_init(&ctx); HMAC_Init_ex(&ctx, key, len, EVP_sha256(), 0)
+-#define HMAC_crunch(ctx, buf, len)    HMAC_Update(&ctx, buf, len)
+-#define HMAC_finish(ctx, dig, dlen)   HMAC_Final(&ctx, dig, &dlen); HMAC_CTX_cleanup(&ctx)
++#if OPENSSL_VERSION_NUMBER < 0x10100000L
++#define HMAC_setup(ctx, key, len)     HMAC_CTX_init(ctx); HMAC_Init_ex(ctx, key, len, EVP_sha256(), 0)
++#else
++#define HMAC_setup(ctx, key, len)     ctx = HMAC_CTX_new(); HMAC_CTX_reset(ctx); HMAC_Init_ex(ctx, key, len, EVP_sha256(), 0)
++#endif
++#define HMAC_crunch(ctx, buf, len)    HMAC_Update(ctx, buf, len)
++#if OPENSSL_VERSION_NUMBER < 0x10100000L
++#define HMAC_finish(ctx, dig, dlen)   HMAC_Final(ctx, dig, &dlen); HMAC_CTX_cleanup(ctx)
++#else
++#define HMAC_finish(ctx, dig, dlen)     HMAC_Final(ctx, dig, &dlen); HMAC_CTX_free(ctx)
++#endif
+ 
+ typedef RC4_KEY *     RC4_handle;
+ #define RC4_alloc(h)  *h = malloc(sizeof(RC4_KEY))
+@@ -117,7 +126,7 @@
+ {
+   uint8_t digest[SHA256_DIGEST_LENGTH];
+   unsigned int digestLen = 0;
+-  HMAC_CTX ctx;
++  HMAC_CTX* ctx = NULL;
+ 
+   RC4_alloc(rc4keyIn);
+   RC4_alloc(rc4keyOut);
+@@ -266,7 +275,7 @@
+          size_t keylen, uint8_t *digest)
+ {
+   unsigned int digestLen;
+-  HMAC_CTX ctx;
++  HMAC_CTX* ctx = NULL;
+ 
+   HMAC_setup(ctx, key, keylen);
+   HMAC_crunch(ctx, message, messageLen);
+--- a/librtmp/hashswf.c
++++ b/librtmp/hashswf.c
+@@ -37,9 +37,9 @@
+ #define SHA256_DIGEST_LENGTH  32
+ #endif
+ #define HMAC_CTX      sha2_context
+-#define HMAC_setup(ctx, key, len)     sha2_hmac_starts(&ctx, (unsigned char *)key, len, 0)
+-#define HMAC_crunch(ctx, buf, len)    sha2_hmac_update(&ctx, buf, len)
+-#define HMAC_finish(ctx, dig, dlen)   dlen = SHA256_DIGEST_LENGTH; sha2_hmac_finish(&ctx, dig)
++#define HMAC_setup(ctx, key, len)     sha2_hmac_starts(ctx, (unsigned char *)key, len, 0)
++#define HMAC_crunch(ctx, buf, len)    sha2_hmac_update(ctx, buf, len)
++#define HMAC_finish(ctx, dig, dlen)   dlen = SHA256_DIGEST_LENGTH; sha2_hmac_finish(ctx, dig)
+ #define HMAC_close(ctx)
+ #elif defined(USE_GNUTLS)
+ #include <nettle/hmac.h>
+@@ -48,19 +48,27 @@
+ #endif
+ #undef HMAC_CTX
+ #define HMAC_CTX      struct hmac_sha256_ctx
+-#define HMAC_setup(ctx, key, len)     hmac_sha256_set_key(&ctx, len, key)
+-#define HMAC_crunch(ctx, buf, len)    hmac_sha256_update(&ctx, len, buf)
+-#define HMAC_finish(ctx, dig, dlen)   dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(&ctx, SHA256_DIGEST_LENGTH, dig)
++#define HMAC_setup(ctx, key, len)     hmac_sha256_set_key(ctx, len, key)
++#define HMAC_crunch(ctx, buf, len)    hmac_sha256_update(ctx, len, buf)
++#define HMAC_finish(ctx, dig, dlen)   dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(ctx, SHA256_DIGEST_LENGTH, dig)
+ #define HMAC_close(ctx)
+ #else /* USE_OPENSSL */
+ #include <openssl/ssl.h>
+ #include <openssl/sha.h>
+ #include <openssl/hmac.h>
+ #include <openssl/rc4.h>
+-#define HMAC_setup(ctx, key, len)     HMAC_CTX_init(&ctx); HMAC_Init_ex(&ctx, (unsigned char *)key, len, EVP_sha256(), 0)
+-#define HMAC_crunch(ctx, buf, len)    HMAC_Update(&ctx, (unsigned char *)buf, len)
+-#define HMAC_finish(ctx, dig, dlen)   HMAC_Final(&ctx, (unsigned char *)dig, &dlen);
+-#define HMAC_close(ctx)       HMAC_CTX_cleanup(&ctx)
++#if OPENSSL_VERSION_NUMBER < 0x10100000L
++#define HMAC_setup(ctx, key, len)     HMAC_CTX_init(ctx); HMAC_Init_ex(ctx, (unsigned char *)key, len, EVP_sha256(), 0)
++#else
++#define HMAC_setup(ctx, key, len)     ctx = HMAC_CTX_new(); HMAC_CTX_reset(ctx); HMAC_Init_ex(ctx, key, len, EVP_sha256(), 0)
++#endif
++#define HMAC_crunch(ctx, buf, len)    HMAC_Update(ctx, (unsigned char *)buf, len)
++#define HMAC_finish(ctx, dig, dlen)   HMAC_Final(ctx, (unsigned char *)dig, &dlen);
++#if OPENSSL_VERSION_NUMBER < 0x10100000L
++#define HMAC_close(ctx)       HMAC_CTX_cleanup(ctx)
++#else
++#define HMAC_close(ctx) HMAC_CTX_reset(ctx); HMAC_CTX_free(ctx)
++#endif
+ #endif
+ 
+ extern void RTMP_TLS_Init();
+@@ -289,7 +297,7 @@
+ struct info
+ {
+   z_stream *zs;
+-  HMAC_CTX ctx;
++  HMAC_CTX *ctx;
+   int first;
+   int zlib;
+   int size;

diff --git a/media-video/rtmpdump/rtmpdump-2.4_p20161210-r3.ebuild b/media-video/rtmpdump/rtmpdump-2.4_p20161210-r3.ebuild
new file mode 100644 (file)
index 0000000..42acf51
--- /dev/null
+++ b/media-video/rtmpdump/rtmpdump-2.4_p20161210-r3.ebuild
@@ -0,0 +1,99 @@
+# Copyright 1999-2019 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI="7"
+
+inherit multilib toolchain-funcs multilib-minimal flag-o-matic
+
+DESCRIPTION="RTMP client intended to stream audio or video flash content"
+HOMEPAGE="https://rtmpdump.mplayerhq.hu/"
+
+# the library is LGPL-2.1, the command is GPL-2
+LICENSE="GPL-2 LGPL-2.1"
+SLOT="0"
+IUSE="gnutls ssl libressl"
+
+DEPEND="ssl? (
+               gnutls? ( >=net-libs/gnutls-2.12.23-r6[${MULTILIB_USEDEP},nettle(+)] )
+               !gnutls? (
+                       !libressl? ( dev-libs/openssl:0=[${MULTILIB_USEDEP}] )
+                       libressl? ( dev-libs/libressl:0=[${MULTILIB_USEDEP}] )
+               )
+               >=sys-libs/zlib-1.2.8-r1[${MULTILIB_USEDEP}]
+       )"
+RDEPEND="${DEPEND}"
+
+PATCHES=(
+       "${FILESDIR}/${PN}-swf_vertification_type_2.patch"
+       "${FILESDIR}/${PN}-swf_vertification_type_2_part_2.patch"
+)
+
+if [[ ${PV} == *9999 ]] ; then
+       KEYWORDS=""
+       SRC_URI=""
+       EGIT_REPO_URI="https://git.ffmpeg.org/rtmpdump.git"
+       inherit git-r3
+else
+       KEYWORDS="~amd64 ~arm ~arm64 ~hppa ~mips ~ppc ~ppc64 ~x86 ~amd64-fbsd ~x86-fbsd ~amd64-linux ~x86-linux"
+       SRC_URI="https://dev.gentoo.org/~hwoarang/distfiles/${P}.tar.gz"
+fi
+
+pkg_setup() {
+       if ! use ssl && use gnutls ; then
+               ewarn "USE='gnutls' is ignored without USE='ssl'."
+               ewarn "Please review the local USE flags for this package."
+       fi
+}
+
+src_unpack() {
+       if [[ ${PV} == *9999 ]] ; then
+               git-r3_src_unpack
+       else
+               mkdir -p "${S}" || die "Can't create source directory"
+               cd "${S}" || die
+               unpack ${A}
+       fi
+}
+
+src_prepare() {
+       # fix #571106 by restoring pre-GCC5 inline semantics
+       append-cflags -std=gnu89
+       # fix Makefile ( bug #298535 , bug #318353 and bug #324513 )
+       sed -i 's/\$(MAKEFLAGS)//g' Makefile \
+               || die "failed to fix Makefile"
+       sed -i -e 's:OPT=:&-fPIC :' \
+               -e 's:OPT:OPTS:' \
+               -e 's:CFLAGS=.*:& $(OPT):' librtmp/Makefile \
+               || die "failed to fix Makefile"
+       use ssl && use !gnutls && use !libressl && eapply "${FILESDIR}/${PN}-openssl-1.1-v2.patch"
+       default
+       multilib_copy_sources
+}
+
+multilib_src_compile() {
+       if use ssl ; then
+               if use gnutls ; then
+                       crypto="GNUTLS"
+               else
+                       crypto="OPENSSL"
+               fi
+       fi
+       #fix multilib-script support. Bug #327449
+       sed -i "/^libdir/s:lib$:$(get_libdir):" librtmp/Makefile || die
+       if ! multilib_is_native_abi; then
+               cd librtmp || die
+       fi
+       emake CC="$(tc-getCC)" LD="$(tc-getLD)" \
+               OPT="${CFLAGS}" XLDFLAGS="${LDFLAGS}" CRYPTO="${crypto}" SYS=posix
+}
+
+multilib_src_install() {
+       mkdir -p "${ED}"/usr/$(get_libdir) || die
+       if multilib_is_native_abi; then
+               dodoc README ChangeLog rtmpdump.1.html rtmpgw.8.html
+       else
+               cd librtmp || die
+       fi
+       emake DESTDIR="${D}" prefix="${EPREFIX}/usr" mandir='$(prefix)/share/man' \
+               CRYPTO="${crypto}" install
+}

diff --git a/media-video/rtmpdump/rtmpdump-9999.ebuild b/media-video/rtmpdump/rtmpdump-9999.ebuild
index 8a45c29d0d07cdbadd6631cc47ae8f3936d35852..42acf5134f909ff3bfee1aeaf7a9e23cebe4acaa 100644 (file)
--- a/media-video/rtmpdump/rtmpdump-9999.ebuild
+++ b/media-video/rtmpdump/rtmpdump-9999.ebuild
@@ -1,7 +1,7 @@
-# Copyright 1999-2018 Gentoo Authors
+# Copyright 1999-2019 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
-EAPI="6"
+EAPI="7"
 
 inherit multilib toolchain-funcs multilib-minimal flag-o-matic
 
@@ -16,8 +16,8 @@ IUSE="gnutls ssl libressl"
 DEPEND="ssl? (
                gnutls? ( >=net-libs/gnutls-2.12.23-r6[${MULTILIB_USEDEP},nettle(+)] )
                !gnutls? (
-                       !libressl? ( >=dev-libs/openssl-1.0.1h-r2[${MULTILIB_USEDEP}] )
-                       libressl? ( dev-libs/libressl )
+                       !libressl? ( dev-libs/openssl:0=[${MULTILIB_USEDEP}] )
+                       libressl? ( dev-libs/libressl:0=[${MULTILIB_USEDEP}] )
                )
                >=sys-libs/zlib-1.2.8-r1[${MULTILIB_USEDEP}]
        )"
@@ -65,7 +65,7 @@ src_prepare() {
                -e 's:OPT:OPTS:' \
                -e 's:CFLAGS=.*:& $(OPT):' librtmp/Makefile \
                || die "failed to fix Makefile"
-       use ssl && use !gnutls && use !libressl && eapply "${FILESDIR}/${PN}-openssl-1.1.patch"
+       use ssl && use !gnutls && use !libressl && eapply "${FILESDIR}/${PN}-openssl-1.1-v2.patch"
        default
        multilib_copy_sources
 }