Signed-off-by: James Almer <jamr...@gmail.com>
---
libavformat/asfcrypt.c | 35 ++++++++++++++++++++++++-----------
libavformat/asfcrypt.h | 2 +-
libavformat/asfdec.c | 12 ++++++++----
3 files changed, 33 insertions(+), 16 deletions(-)
diff --git a/libavformat/asfcrypt.c b/libavformat/asfcrypt.c
index c261475..8292eae 100644
--- a/libavformat/asfcrypt.c
+++ b/libavformat/asfcrypt.c
@@ -144,35 +144,42 @@ static uint64_t multiswap_dec(const uint32_t keys[12],
return ((uint64_t)b << 32) | a;
}
-void ff_asfcrypt_dec(const uint8_t key[20], uint8_t *data, int len)
+int ff_asfcrypt_dec(const uint8_t key[20], uint8_t *data, int len)
{
- struct AVDES des;
- struct AVRC4 rc4;
+ struct AVDES *des;
+ struct AVRC4 *rc4;
int num_qwords = len >> 3;
uint8_t *qwords = data;
uint64_t rc4buff[8] = { 0 };
uint64_t packetkey;
uint32_t ms_keys[12];
uint64_t ms_state;
- int i;
+ int i, ret = 0;
if (len < 16) {
for (i = 0; i < len; i++)
data[i] ^= key[i];
- return;
+ return ret;
}
- av_rc4_init(&rc4, key, 12 * 8, 1);
- av_rc4_crypt(&rc4, (uint8_t *)rc4buff, NULL, sizeof(rc4buff), NULL, 1);
+ if (!(rc4 = av_rc4_alloc()))
+ return AVERROR(ENOMEM);
+ av_rc4_init(rc4, key, 12 * 8, 1);
+ av_rc4_crypt(rc4, (uint8_t *)rc4buff, NULL, sizeof(rc4buff), NULL, 1);
multiswap_init((uint8_t *)rc4buff, ms_keys);
packetkey = AV_RN64(&qwords[num_qwords * 8 - 8]);
packetkey ^= rc4buff[7];
- av_des_init(&des, key + 12, 64, 1);
- av_des_crypt(&des, (uint8_t *)&packetkey, (uint8_t *)&packetkey, 1, NULL,
1);
+
+ if (!(des = av_des_alloc())) {
+ ret = AVERROR(ENOMEM);
+ goto fail;
+ }
+ av_des_init(des, key + 12, 64, 1);
+ av_des_crypt(des, (uint8_t *)&packetkey, (uint8_t *)&packetkey, 1, NULL,
1);
packetkey ^= rc4buff[6];
- av_rc4_init(&rc4, (uint8_t *)&packetkey, 64, 1);
- av_rc4_crypt(&rc4, data, data, len, NULL, 1);
+ av_rc4_init(rc4, (uint8_t *)&packetkey, 64, 1);
+ av_rc4_crypt(rc4, data, data, len, NULL, 1);
ms_state = 0;
for (i = 0; i < num_qwords - 1; i++, qwords += 8)
@@ -182,4 +189,10 @@ void ff_asfcrypt_dec(const uint8_t key[20], uint8_t *data,
int len)
packetkey = av_le2ne64(packetkey);
packetkey = multiswap_dec(ms_keys, ms_state, packetkey);
AV_WL64(qwords, packetkey);
+
+fail:
+ av_freep(&rc4);
+ av_freep(&des);
+
+ return ret;
}
diff --git a/libavformat/asfcrypt.h b/libavformat/asfcrypt.h
index 53388b4..d4f311c 100644
--- a/libavformat/asfcrypt.h
+++ b/libavformat/asfcrypt.h
@@ -24,6 +24,6 @@
#include <inttypes.h>
-void ff_asfcrypt_dec(const uint8_t key[20], uint8_t *data, int len);
+int ff_asfcrypt_dec(const uint8_t key[20], uint8_t *data, int len);
#endif /* AVFORMAT_ASFCRYPT_H */
diff --git a/libavformat/asfdec.c b/libavformat/asfdec.c
index 37d91e0..058bfa2 100644
--- a/libavformat/asfdec.c
+++ b/libavformat/asfdec.c
@@ -1167,8 +1167,10 @@ static int asf_read_multiple_payload(AVFormatContext *s,
AVPacket *pkt,
}
if ((ret = avio_read(pb, p, pay_len)) < 0)
return ret;
- if (s->key && s->keylen == 20)
- ff_asfcrypt_dec(s->key, p, ret);
+ if (s->key && s->keylen == 20) {
+ if ((ret = ff_asfcrypt_dec(s->key, p, ret)) < 0)
+ return ret;
+ }
avio_skip(pb, skip);
asf_pkt->size_left -= pay_len;
asf->nb_mult_left--;
@@ -1220,8 +1222,10 @@ static int asf_read_single_payload(AVFormatContext *s,
AVPacket *pkt,
asf_pkt->size_left = 0;
if ((ret = avio_read(pb, p, size)) < 0)
return ret;
- if (s->key && s->keylen == 20)
- ff_asfcrypt_dec(s->key, p, ret);
+ if (s->key && s->keylen == 20) {
+ if ((ret = ff_asfcrypt_dec(s->key, p, ret)) < 0)
+ return ret;
+ }
if (asf->packet_size_internal)
avio_skip(pb, asf->packet_size - asf->packet_size_internal);
avio_skip(pb, asf->pad_len); // skip padding
--
2.4.5
_______________________________________________
libav-devel mailing list
libav-devel@libav.org
https://lists.libav.org/mailman/listinfo/libav-devel
