On Fri, May 23, 2014 at 10:10 AM, Satoshi Nakagawa <[email protected]> wrote: > # HG changeset patch > # User Satoshi Nakagawa <[email protected]> > # Date 1400857636 -32400 > # Sat May 24 00:07:16 2014 +0900 > # Node ID 61ad3e4d5b3264f7d6b12f2820292f53cb98cbbd > # Parent 5134e76aa729b6fece18701fdc00390c2f2ffb32 > rdcost: overflow check by integer > > diff -r 5134e76aa729 -r 61ad3e4d5b32 source/encoder/rdcost.h > --- a/source/encoder/rdcost.h Thu May 22 21:46:21 2014 -0500 > +++ b/source/encoder/rdcost.h Sat May 24 00:07:16 2014 +0900 > @@ -26,6 +26,11 @@ > > #include "common.h" > > +#if !defined(UINT64_MAX) > +#include <limits> > +#define UINT64_MAX (std::numeric_limits<uint64_t>::max()) > +#endif
this patch is great except this bit. UINT64_MAX is supposed to be defined by stdint.h, which we include in common.h http://tigcc.ticalc.org/doc/stdint.html If it was missing for you, it likely means we need to add this hack to compat/msvc/stdint.h > namespace x265 { > // private namespace > > @@ -75,8 +80,7 @@ > > inline uint64_t calcRdCost(uint32_t distortion, uint32_t bits) > { > - X265_CHECK(abs((float)((bits * m_lambdaSSE + 128) >> 8) - > - (float)bits * m_lambdaSSE / 256.0) < 2, > + X265_CHECK(bits <= (UINT64_MAX - 128) / m_lambdaSSE, > "calcRdCost wrap detected dist: %d, bits %d, lambda: > %d\n", distortion, bits, (int)m_lambdaSSE); > return distortion + ((bits * m_lambdaSSE + 128) >> 8); > } > @@ -105,8 +109,7 @@ > > inline uint64_t calcRdSADCost(uint32_t sadCost, uint32_t bits) > { > - X265_CHECK(abs((float)((bits * m_lambdaSAD + 128) >> 8) - > - (float)bits * m_lambdaSAD / 256.0) < 2, > + X265_CHECK(bits <= (UINT64_MAX - 128) / m_lambdaSAD, > "calcRdSADCost wrap detected dist: %d, bits %d, lambda: > "X265_LL"\n", sadCost, bits, m_lambdaSAD); > return sadCost + ((bits * m_lambdaSAD + 128) >> 8); > } > @@ -118,16 +121,14 @@ > > inline uint32_t scaleChromaDistCb(uint32_t dist) > { > - X265_CHECK(abs((float)((dist * m_cbDistortionWeight + 128) >> 8) - > - (float)dist * m_cbDistortionWeight / 256.0) < 2, > + X265_CHECK(dist <= (UINT64_MAX - 128) / m_cbDistortionWeight, > "scaleChromaDistCb wrap detected dist: %d, lambda: > "X265_LL"\n", dist, m_cbDistortionWeight); > return (uint32_t)(((dist * m_cbDistortionWeight) + 128) >> 8); > } > > inline uint32_t scaleChromaDistCr(uint32_t dist) > { > - X265_CHECK(abs((float)((dist * m_crDistortionWeight + 128) >> 8) - > - (float)dist * m_crDistortionWeight / 256.0) < 2, > + X265_CHECK(dist <= (UINT64_MAX - 128) / m_crDistortionWeight, > "scaleChromaDistCr wrap detected dist: %d, lambda: > "X265_LL"\n", dist, m_crDistortionWeight); > return (uint32_t)(((dist * m_crDistortionWeight) + 128) >> 8); > } > _______________________________________________ > x265-devel mailing list > [email protected] > https://mailman.videolan.org/listinfo/x265-devel -- Steve Borho _______________________________________________ x265-devel mailing list [email protected] https://mailman.videolan.org/listinfo/x265-devel
