RPM Package Manager, CVS Repository http://rpm5.org/cvs/ ____________________________________________________________________________
Server: rpm5.org Name: Jeff Johnson Root: /v/rpm/cvs Email: [email protected] Module: rpm Date: 04-Jun-2017 06:45:52 Branch: rpm-5_4 Handle: 2017060404455200 Modified files: (Branch: rpm-5_4) rpm/rpmio rpmzstd.c Log: - zstd: WIP. Summary: Revision Changes Path 1.1.2.3 +180 -56 rpm/rpmio/rpmzstd.c ____________________________________________________________________________ patch -p0 <<'@@ .' Index: rpm/rpmio/rpmzstd.c ============================================================================ $ cvs diff -u -r1.1.2.2 -r1.1.2.3 rpmzstd.c --- rpm/rpmio/rpmzstd.c 4 Jun 2017 02:24:33 -0000 1.1.2.2 +++ rpm/rpmio/rpmzstd.c 4 Jun 2017 04:45:52 -0000 1.1.2.3 @@ -16,6 +16,8 @@ #include "system.h" #include <rpmdefs.h> +#include <rpmmacro.h> +#include <argv.h> #if defined(POPTIO_OPTION_PROCESSING) # include <poptIO.h> #endif /* POPTIO_OPTION_PROCESSING */ @@ -30,6 +32,11 @@ #undef MAX #define MAX(a,b) ((a) > (b) ? (a) : (b)) +static int _debug = -1; + +#define SPEW(_fmt, ...) \ + if (_debug) fprintf(stderr, _fmt, __VA_ARGS__) + /*==============================================================*/ /* --- platform.h */ @@ -1202,10 +1209,10 @@ } -UTIL_STATIC U64 UTIL_getTotalFileSize(const char ** av, unsigned ac) +UTIL_STATIC U64 UTIL_getTotalFileSize(ARGV_t av, int ac) { U64 total = 0; - for (unsigned n = 0; n < ac; n++) + for (int n = 0; n < ac; n++) total += UTIL_getFileSize(av[n]); return total; } @@ -1428,7 +1435,7 @@ } -UTIL_STATIC void UTIL_freeFileList(const char ** av, char * avbuf) +UTIL_STATIC void UTIL_freeFileList(ARGV_t av, char * avbuf) { if (avbuf) free(avbuf); if (av) free((void *)av); @@ -1699,14 +1706,14 @@ ***************************************/ /** FIO_compressMultipleFilenames() : @return : nb of missing files */ -int FIO_compressMultipleFilenames(const char ** av, unsigned ac, +int FIO_compressMultipleFilenames(ARGV_t av, int ac, const char * suffix, const char * dfn, int compressionLevel, ZSTD_compressionParameters* comprParams); /** FIO_decompressMultipleFilenames() : @return : nb of missing or skipped files */ -int FIO_decompressMultipleFilenames(const char ** av, unsigned ac, +int FIO_decompressMultipleFilenames(ARGV_t av, int ac, const char * suffix, const char * dfn); @@ -1736,7 +1743,7 @@ #ifndef ZSTD_NOBENCH -int BMK_benchFiles(const char ** av, unsigned ac, const char * dfn, +int BMK_benchFiles(ARGV_t av, int ac, const char * dfn, int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio); /* Set Parameters */ @@ -1775,7 +1782,7 @@ @return : 0 == ok. Any other : error. */ int DiB_trainFromFiles(const char * dfn, unsigned maxDictSize, - const char ** av, unsigned ac, + ARGV_t av, int ac, ZDICT_params_t *params, COVER_params_t *coverParams, int optimizeCover); @@ -1802,6 +1809,7 @@ #define RDG_rotl32(x,r) ((x << r) | (x >> (32 - r))) static U32 RDG_rand(U32* src) { +SPEW("-->\t%s()\n", __FUNCTION__); static const U32 prime1 = 2654435761U; static const U32 prime2 = 2246822519U; U32 rand32 = *src; @@ -1833,6 +1841,7 @@ static BYTE RDG_genChar(U32* seed, const BYTE* ldt) { +SPEW("-->\t%s()\n", __FUNCTION__); U32 const id = RDG_rand(seed) & LTMASK; return ldt[id]; /* memory-sanitizer fails here, stating "uninitialized value" when table initialized with P == 0.0. Checked : table is fully initialized */ } @@ -1840,17 +1849,20 @@ static U32 RDG_rand15Bits (unsigned* seedPtr) { +SPEW("-->\t%s()\n", __FUNCTION__); return RDG_rand(seedPtr) & 0x7FFF; } static U32 RDG_randLength(unsigned* seedPtr) { +SPEW("-->\t%s()\n", __FUNCTION__); if (RDG_rand(seedPtr) & 7) return (RDG_rand(seedPtr) & 0xF); /* small length */ return (RDG_rand(seedPtr) & 0x1FF) + 0xF; } void RDG_genBlock(void * buffer, size_t buffSize, size_t prefixSize, double matchProba, const BYTE* ldt, unsigned* seedPtr) { +SPEW("-->\t%s()\n", __FUNCTION__); BYTE* const buffPtr = (BYTE*)buffer; U32 const matchProba32 = (U32)(32768 * matchProba); size_t pos = prefixSize; @@ -1898,6 +1910,7 @@ void RDG_genBuffer(void * buffer, size_t size, double matchProba, double litProba, unsigned seed) { +SPEW("-->\t%s()\n", __FUNCTION__); BYTE ldt[LTSIZE]; memset(ldt, '0', sizeof(ldt)); /* yes, character '0', this is intentional */ if (litProba<=0.0) litProba = matchProba / 4.5; @@ -1908,6 +1921,7 @@ void RDG_genStdout(unsigned long long size, double matchProba, double litProba, unsigned seed) { +SPEW("-->\t%s()\n", __FUNCTION__); size_t const stdBlockSize = 128 KB; size_t const stdDictSize = 32 KB; BYTE* const buff = (BYTE*)malloc(stdDictSize + stdBlockSize); @@ -2022,6 +2036,7 @@ static U32 g_nbSeconds = BMK_TIMETEST_DEFAULT_S; void BMK_setNbSeconds(unsigned nbSeconds) { +SPEW("-->\t%s()\n", __FUNCTION__); g_nbSeconds = nbSeconds; DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression - \n", g_nbSeconds); } @@ -2029,6 +2044,7 @@ static size_t g_blockSize = 0; void BMK_setBlockSize(size_t blockSize) { +SPEW("-->\t%s()\n", __FUNCTION__); g_blockSize = blockSize; if (g_blockSize) DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10)); } @@ -2036,7 +2052,9 @@ void BMK_setDecodeOnlyMode(unsigned decodeFlag) { g_decodeOnly = (decodeFlag>0); } static U32 g_nbThreads = 1; -void BMK_setNbThreads(unsigned nbThreads) { +void BMK_setNbThreads(unsigned nbThreads) +{ +SPEW("-->\t%s()\n", __FUNCTION__); #ifndef ZSTD_MULTITHREAD if (nbThreads > 1) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n"); #endif @@ -2063,6 +2081,7 @@ const void * dictBuffer, size_t dictBufferSize, const ZSTD_compressionParameters* comprParams) { +SPEW("-->\t%s()\n", __FUNCTION__); size_t const blockSize = ((g_blockSize>=32 && !g_decodeOnly) ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; size_t const avgSize = MIN(blockSize, (srcSize / nbFiles)); U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; @@ -2328,12 +2347,14 @@ ZSTDMT_freeCCtx(mtctx); ZSTD_freeCCtx(ctx); ZSTD_freeDCtx(dctx); +SPEW("<--\t%s() rc %d\n", __FUNCTION__, 0); return 0; } static size_t BMK_findMaxMem(U64 requiredMem) { +SPEW("-->\t%s()\n", __FUNCTION__); size_t const step = 64 MB; BYTE* testmem = NULL; @@ -2347,6 +2368,7 @@ } while (!testmem); free(testmem); +SPEW("<--\t%s() rc %llu\n", __FUNCTION__, requiredMem); return (size_t)(requiredMem); } @@ -2356,6 +2378,7 @@ const void * dictBuffer, size_t dictBufferSize, ZSTD_compressionParameters *compressionParams, int setRealTimePrio) { +SPEW("-->\t%s()\n", __FUNCTION__); int l; const char * pch = strrchr(displayName, '\\'); /* Windows */ @@ -2386,8 +2409,9 @@ At most, fills `buffer` entirely */ static void BMK_loadFiles(void * buffer, size_t bufferSize, size_t* fileSizes, - const char ** av, unsigned ac) + ARGV_t av, int ac) { +SPEW("-->\t%s()\n", __FUNCTION__); size_t totalSize = 0; size_t pos = 0; for (unsigned n = 0; n < ac; n++) { @@ -2417,7 +2441,7 @@ if (totalSize == 0) EXM_THROW(12, "no data to bench"); } -static void BMK_benchFileTable(const char ** av, unsigned ac, const char * dfn, int cLevel, +static void BMK_benchFileTable(ARGV_t av, int ac, const char * dfn, int cLevel, int cLevelLast, ZSTD_compressionParameters *compressionParams, int setRealTimePrio) { void * srcBuffer; @@ -2469,6 +2493,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams, int setRealTimePrio) { +SPEW("-->\t%s()\n", __FUNCTION__); char name[20] = {0}; size_t benchedSize = 10000000; void * const srcBuffer = malloc(benchedSize); @@ -2488,9 +2513,10 @@ } -int BMK_benchFiles(const char ** av, unsigned ac, const char * dfn, +int BMK_benchFiles(ARGV_t av, int ac, const char * dfn, int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio) { +SPEW("-->\t%s()\n", __FUNCTION__); double const compressibility = (double)g_compressibilityDefault / 100; if (cLevel < 1) cLevel = 1; /* minimum compression level */ @@ -2503,6 +2529,7 @@ BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams, setRealTimePrio); else BMK_benchFileTable(av, ac, dfn, cLevel, cLevelLast, compressionParams, setRealTimePrio); +SPEW("<--\t%s() rc %u\n", __FUNCTION__, 0); return 0; } @@ -2582,12 +2609,14 @@ * @return : nb of files effectively loaded into `buffer` */ static unsigned DiB_loadFiles(void * buffer, size_t* bufferSizePtr, size_t* fileSizes, - const char ** av, unsigned ac) + ARGV_t av, int ac) { +SPEW("-->\t%s()\n", __FUNCTION__); char * const buff = (char *)buffer; size_t pos = 0; + unsigned n; - for (unsigned n = 0; n < ac; n++) { + for (n = 0; n < ac; n++) { const char * const fn = av[n]; unsigned long long const fs64 = UTIL_getFileSize(fn); size_t const fileSize = (size_t) MIN(fs64, SAMPLESIZE_MAX); @@ -2603,6 +2632,7 @@ } } DISPLAYLEVEL(2, "\r%79s\r", ""); *bufferSizePtr = pos; +SPEW("<--\t%s() rc %u\n", __FUNCTION__, n); return n; } @@ -2616,10 +2646,13 @@ rand32 ^= prime2; rand32 = DiB_rotl32(rand32, 13); *src = rand32; +SPEW("<--\t%s() rc 0x%x\n", __FUNCTION__, rand32 >> 5); return rand32 >> 5; } -static void DiB_shuffle(const char ** av, unsigned ac) { +static void DiB_shuffle(ARGV_t av, int ac) +{ +SPEW("-->\t%s()\n", __FUNCTION__); /* Initialize the pseudorandom number generator */ U32 seed = 0xFD2FB528; for (unsigned i = ac - 1; i > 0; --i) { @@ -2636,6 +2669,7 @@ **********************************************************/ static size_t DiB_findMaxMem(unsigned long long requiredMem) { +SPEW("-->\t%s()\n", __FUNCTION__); size_t const step = 8 MB; void * testmem = NULL; @@ -2649,12 +2683,14 @@ } free(testmem); +SPEW("<--\t%s() rc %llx\n", __FUNCTION__, requiredMem); return (size_t)requiredMem; } static void DiB_fillNoise(void * buffer, size_t length) { +SPEW("-->\t%s()\n", __FUNCTION__); unsigned const prime1 = 2654435761U; unsigned const prime2 = 2246822519U; unsigned acc = prime1; @@ -2670,6 +2706,7 @@ static void DiB_saveDict(const char * dfn, const void * buff, size_t buffSize) { +SPEW("-->\t%s()\n", __FUNCTION__); FILE * const f = fopen(dfn, "wb"); if (f == NULL) EXM_THROW(3, "cannot open %s ", dfn); @@ -2682,8 +2719,9 @@ static int g_tooLargeSamples = 0; -static U64 DiB_getTotalCappedFileSize(const char ** av, unsigned ac) +static U64 DiB_getTotalCappedFileSize(ARGV_t av, int ac) { +SPEW("-->\t%s()\n", __FUNCTION__); U64 total = 0; for (unsigned n = 0; n < ac; n++) { U64 const fileSize = UTIL_getFileSize(av[n]); @@ -2691,6 +2729,7 @@ total += cappedFileSize; g_tooLargeSamples |= (fileSize > 2*SAMPLESIZE_MAX); } +SPEW("<--\t%s() rc %llu\n", __FUNCTION__, total); return total; } @@ -2708,10 +2747,11 @@ int DiB_trainFromFiles(const char * dfn, unsigned maxDictSize, - const char ** av, unsigned ac, + ARGV_t av, int ac, ZDICT_params_t *params, COVER_params_t *coverParams, int optimizeCover) { +SPEW("-->\t%s()\n", __FUNCTION__); void * const dictBuffer = malloc(maxDictSize); size_t * const fileSizes = (size_t*)malloc(ac * sizeof(size_t)); unsigned long long const totalSizeToLoad = DiB_getTotalCappedFileSize(av, ac); @@ -2782,6 +2822,7 @@ free(srcBuffer); free(dictBuffer); free(fileSizes); +SPEW("<--\t%s() rc %p\n", __FUNCTION__, result); return result; } @@ -2936,7 +2977,9 @@ } #define FIO_OVERLAP_LOG_NOTSET 9999 static U32 g_FIO_overlapLog = FIO_OVERLAP_LOG_NOTSET; -void FIO_setOverlapLog(unsigned overlapLog){ +void FIO_setOverlapLog(unsigned overlapLog) +{ +SPEW("-->\t%s()\n", __FUNCTION__); if (overlapLog && g_nbThreads == 1) DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n"); g_FIO_overlapLog = overlapLog; @@ -2967,12 +3010,16 @@ * @result : Unlink `fileName`, even if it's read-only */ static int FIO_remove(const char * path) { +SPEW("-->\t%s(%s)\n", __FUNCTION__, path); + int rc; #if defined(_WIN32) || defined(WIN32) /* windows doesn't allow remove read-only files, * so try to make it writable first */ chmod(path, _S_IWRITE); #endif - return remove(path); + rc = remove(path); +SPEW("<--\t%s(%s) rc %d\n", __FUNCTION__, path, rc); + return rc; } /** FIO_openSrcFile() : @@ -2980,6 +3027,7 @@ * @result : FILE * to `dstfn`, or NULL if it fails */ static FILE * FIO_openSrcFile(const char * srcfn) { +SPEW("-->\t%s(%s)\n", __FUNCTION__, srcfn); FILE * fp; if (!strcmp (srcfn, stdinmark)) { @@ -2995,6 +3043,7 @@ if (fp == NULL) DISPLAYLEVEL(1, "zstd: %s: %s \n", srcfn, strerror(errno)); } +SPEW("<--\t%s(%s) fp %p\n", __FUNCTION__, srcfn, fp); return fp; } @@ -3003,6 +3052,7 @@ * @result : FILE * to `dstfn`, or NULL if it fails */ static FILE * FIO_openDstFile(const char * dstfn) { +SPEW("-->\t%s(%s)\n", __FUNCTION__, dstfn); FILE * fp; if (!strcmp (dstfn, stdoutmark)) { @@ -3044,6 +3094,7 @@ if (fp == NULL) DISPLAYLEVEL(1, "zstd: %s: %s\n", dstfn, strerror(errno)); } +SPEW("<--\t%s() fp %p\n", __FUNCTION__, fp); return fp; } @@ -3056,6 +3107,7 @@ */ static size_t FIO_createDictBuffer(void ** bufferPtr, const char * fileName) { +SPEW("-->\t%s(%p, %s)\n", __FUNCTION__, bufferPtr, fileName); FILE * fp; U64 fileSize; @@ -3072,6 +3124,7 @@ { size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fp); if (readSize!=fileSize) EXM_THROW(35, "Error reading dictionary file %s", fileName); } fclose(fp); +SPEW("<--\t%s(%p, %s) rc %zd\n", __FUNCTION__, bufferPtr, fileName, (size_t)fileSize); return (size_t)fileSize; } @@ -3098,6 +3151,7 @@ U64 srcSize, int srcRegFile, ZSTD_compressionParameters* comprParams) { +SPEW("-->\t%s(%s, %d, %llu, %d, %p)\n", __FUNCTION__, dfn, cLevel, (unsigned long long)srcSize, srcRegFile, comprParams); cRess_t ress; memset(&ress, 0, sizeof(ress)); @@ -3150,11 +3204,13 @@ free(dictBuffer); } +SPEW("<--\t%s(%s, %d, %llu, %d, %p) ress %p\n", __FUNCTION__, dfn, cLevel, (unsigned long long)srcSize, srcRegFile, comprParams, (void *)&ress); return ress; } static void FIO_freeCResources(cRess_t ress) { +SPEW("-->\t%s(%p)\n", __FUNCTION__, &ress); free(ress.srcBuffer); free(ress.dstBuffer); #ifdef ZSTD_MULTITHREAD @@ -3170,6 +3226,7 @@ const char * srcfn, U64 const srcFileSize, int compressionLevel, U64* readsize) { +SPEW("-->\t%s()\n", __FUNCTION__); unsigned long long inFileSize = 0, outFileSize = 0; z_stream strm; int ret; @@ -3242,6 +3299,7 @@ EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcfn, ret); *readsize = inFileSize; +SPEW("<--\t%s() rc %llu\n", __FUNCTION__, outFileSize); return outFileSize; } #endif @@ -3252,6 +3310,7 @@ const char * srcfn, U64 const srcFileSize, int compressionLevel, U64* readsize, int plain_lzma) { +SPEW("-->\t%s()\n", __FUNCTION__); unsigned long long inFileSize = 0, outFileSize = 0; lzma_stream strm = LZMA_STREAM_INIT; lzma_action action = LZMA_RUN; @@ -3313,6 +3372,7 @@ lzma_end(&strm); *readsize = inFileSize; +SPEW("<--\t%s() rc %llu\n", __FUNCTION__, outFileSize); return outFileSize; } #endif @@ -3323,7 +3383,9 @@ const char * srcfn, U64 const srcFileSize, int compressionLevel, U64* readsize) { - unsigned long long inFileSize = 0, outFileSize = 0; +SPEW("-->\t%s()\n", __FUNCTION__); + unsigned long long inFileSize = 0; + unsigned long long outFileSize = 0; LZ4F_preferences_t prefs; LZ4F_compressionContext_t ctx; @@ -3406,6 +3468,7 @@ *readsize = inFileSize; LZ4F_freeCompressionContext(ctx); +SPEW("<--\t%s() rc %llu\n", __FUNCTION__, outFileSize); return outFileSize; } #endif @@ -3419,6 +3482,7 @@ static int FIO_compressFilename_internal(cRess_t ress, const char * dstfn, const char * srcfn, int compressionLevel) { +SPEW("-->\t%s(%p, %s, %s)\n", __FUNCTION__, &ress, dstfn, srcfn); FILE * const srcfp = ress.srcfp; FILE * const dstfp = ress.dstfp; U64 readsize = 0; @@ -3539,6 +3603,7 @@ (unsigned long long)readsize, (unsigned long long) compressedfilesize, dstfn); +SPEW("<--\t%s(%p, %s, %s) rc %d\n", __FUNCTION__, &ress, dstfn, srcfn, 0); return 0; } @@ -3552,6 +3617,7 @@ const char * dstfn, const char * srcfn, int compressionLevel) { +SPEW("-->\t%s(%p, %s, %s, %d)\n", __FUNCTION__, &ress, dstfn, srcfn, compressionLevel); int result; /* File check */ @@ -3570,6 +3636,7 @@ if (remove(srcfn)) EXM_THROW(1, "zstd: %s: %s", srcfn, strerror(errno)); } +SPEW("<--\t%s(%p, %s, %s, %d) rc %d\n", __FUNCTION__, &ress, dstfn, srcfn, compressionLevel, result); return result; } @@ -3583,6 +3650,7 @@ const char * srcfn, int compressionLevel) { +SPEW("-->\t%s(%p, %s, %s, %d)\n", __FUNCTION__, &ress, dstfn, srcfn, compressionLevel); int result; stat_t sb; int stat_result = 0; @@ -3604,6 +3672,7 @@ } else if (strcmp (dstfn, stdoutmark) && stat_result) UTIL_setFileStat(dstfn, &sb); +SPEW("<--\t%s(%p, %s, %s, %d) rc %d\n", __FUNCTION__, &ress, dstfn, srcfn, compressionLevel, result); return result; } @@ -3611,6 +3680,7 @@ int FIO_compressFilename(const char * dstfn, const char * srcfn, const char * dfn, int compressionLevel, ZSTD_compressionParameters* comprParams) { +SPEW("-->\t%s()\n", __FUNCTION__); clock_t const start = clock(); U64 const srcSize = UTIL_getFileSize(srcfn); int const regFile = UTIL_isRegFile(srcfn); @@ -3622,15 +3692,17 @@ DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds); FIO_freeCResources(ress); +SPEW("<--\t%s() rc %d\n", __FUNCTION__, result); return result; } -int FIO_compressMultipleFilenames(const char ** av, unsigned ac, +int FIO_compressMultipleFilenames(ARGV_t av, int ac, const char * suffix, const char * dfn, int compressionLevel, ZSTD_compressionParameters* comprParams) { +SPEW("-->\t%s(%p[%u], %s, %s, %d, %p)\n", __FUNCTION__, av, ac, suffix, dfn, compressionLevel, comprParams); int missed_files = 0; size_t dfnSize = FNSPACE; char * dstfn = (char *)malloc(FNSPACE); @@ -3649,13 +3721,13 @@ if (!strcmp(suffix, stdoutmark)) { ress.dstfp = stdout; SET_BINARY_MODE(stdout); - for (unsigned u = 0; u < ac; u++) - missed_files += FIO_compressFilename_srcFile(ress, stdoutmark, av[u], compressionLevel); + for (int i = 0; i < ac; i++) + missed_files += FIO_compressFilename_srcFile(ress, stdoutmark, av[i], compressionLevel); if (fclose(ress.dstfp)) EXM_THROW(29, "Write error : cannot properly close stdout"); } else { - for (unsigned u = 0; u < ac; u++) { - size_t const ifnSize = strlen(av[u]); + for (int i = 0; i < ac; i++) { + size_t const ifnSize = strlen(av[i]); if (dfnSize <= ifnSize+suffixSize+1) { /* resize name buffer */ free(dstfn); dfnSize = ifnSize + 20; @@ -3663,15 +3735,16 @@ if (!dstfn) EXM_THROW(30, "zstd: %s", strerror(errno)); } - strcpy(dstfn, av[u]); + strcpy(dstfn, av[i]); strcat(dstfn, suffix); - missed_files += FIO_compressFilename_dstFile(ress, dstfn, av[u], compressionLevel); + missed_files += FIO_compressFilename_dstFile(ress, dstfn, av[i], compressionLevel); } } /* Close & Free */ FIO_freeCResources(ress); free(dstfn); +SPEW("<--\t%s(%p[%u], %s, %s, %d, %p) missed_files %d\n", __FUNCTION__, av, ac, suffix, dfn, compressionLevel, comprParams, missed_files); return missed_files; } @@ -3696,6 +3769,7 @@ static dRess_t FIO_createDResources(const char * dfn) { +SPEW("-->\t%s(%s)\n", __FUNCTION__, dfn); dRess_t ress; memset(&ress, 0, sizeof(ress)); @@ -3719,11 +3793,13 @@ free(dictBuffer); } +SPEW("<--\t%s(%s) rc %p\n", __FUNCTION__, dfn, (void *)&ress); return ress; } static void FIO_freeDResources(dRess_t ress) { +SPEW("-->\t%s(%p)\n", __FUNCTION__, &ress); size_t const errorCode = ZSTD_freeDStream(ress.dctx); if (ZSTD_isError(errorCode)) EXM_THROW(69, "Error : can't free ZSTD_DStream context resource : %s", @@ -3737,6 +3813,7 @@ * @return : storedSkips, to be provided to next call to FIO_fwriteSparse() of LZ4IO_fwriteSparseEnd() */ static unsigned FIO_fwriteSparse(FILE * fp, const void * buffer, size_t bufferSize, unsigned storedSkips) { +SPEW("-->\t%s(%p, %p[%zd], %u)\n", __FUNCTION__, fp, buffer, bufferSize, storedSkips); const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ size_t bufferSizeT = bufferSize / sizeof(size_t); const size_t* const bufferTEnd = bufferT + bufferSizeT; @@ -3798,11 +3875,13 @@ EXM_THROW(75, "Write error : cannot write decoded end of block"); } } } } +SPEW("<--\t%s(%p, %p[%zd], %u) storedSkips %u\n", __FUNCTION__, fp, buffer, bufferSize, storedSkips, storedSkips); return storedSkips; } static void FIO_fwriteSparseEnd(FILE * fp, unsigned storedSkips) { +SPEW("-->\t%s(%p, %u)\n", __FUNCTION__, fp, storedSkips); if (storedSkips-->0) { /* implies g_sparseFileSupport>0 */ int const seekResult = LONG_SEEK(fp, storedSkips, SEEK_CUR); if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)"); @@ -3818,6 +3897,7 @@ @return : 0 (no error) */ static unsigned FIO_passThrough(FILE * ofp, FILE * ifp, void * buffer, size_t bufferSize, size_t alreadyLoaded) { +SPEW("-->\t%s()\n", __FUNCTION__); size_t const blockSize = MIN(64 KB, bufferSize); size_t readFromInput = 1; unsigned storedSkips = 0; @@ -3832,6 +3912,7 @@ } FIO_fwriteSparseEnd(ofp, storedSkips); +SPEW("<--\t%s() rc %d\n", __FUNCTION__, 0); return 0; } @@ -3845,6 +3926,7 @@ const char * srcfn, U64 alreadyDecoded) { +SPEW("-->\t%s(%p, %p, %s, %llu)\n", __FUNCTION__, ress, ifp, srcfn, (unsigned long long)alreadyDecoded); U64 frameSize = 0; U32 storedSkips = 0; @@ -3899,6 +3981,7 @@ FIO_fwriteSparseEnd(ress->dstfp, storedSkips); +SPEW("<--\t%s(%p, %p, %s, %llu) rc %llu\n", __FUNCTION__, ress, ifp, srcfn, (unsigned long long)alreadyDecoded, (unsigned long long)frameSize); return frameSize; } @@ -3907,6 +3990,7 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE * srcfp, const char * srcfn) { +SPEW("-->\t%s()\n", __FUNCTION__); unsigned long long outFileSize = 0; z_stream strm; int flush = Z_NO_FLUSH; @@ -3955,6 +4039,7 @@ ress->srcBufferLoaded = strm.avail_in; ret = inflateEnd(&strm); if (ret != Z_OK) EXM_THROW(32, "zstd: %s: inflateEnd error %d", srcfn, ret); +SPEW("<--\t%s() rc %llu\n", __FUNCTION__, outFileSize); return outFileSize; } #endif @@ -3963,6 +4048,7 @@ #ifdef ZSTD_LZMADECOMPRESS static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE * srcfp, const char * srcfn, int plain_lzma) { +SPEW("-->\t%s()\n", __FUNCTION__); unsigned long long outFileSize = 0; lzma_stream strm = LZMA_STREAM_INIT; lzma_action action = LZMA_RUN; @@ -4014,6 +4100,7 @@ if (strm.avail_in > 0) memmove(ress->srcBuffer, strm.next_in, strm.avail_in); ress->srcBufferLoaded = strm.avail_in; lzma_end(&strm); +SPEW("<--\t%s() rc %llu\n", __FUNCTION__, outFileSize); return outFileSize; } #endif @@ -4022,6 +4109,7 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress, FILE * srcfp, const char * srcfn) { +SPEW("-->\t%s()\n", __FUNCTION__); unsigned long long filesize = 0; LZ4F_errorCode_t nextToLoad; LZ4F_decompressionContext_t dCtx; @@ -4080,6 +4168,7 @@ LZ4F_freeDecompressionContext(dCtx); ress->srcBufferLoaded = 0; /* LZ4F will go to the frame boundary */ +SPEW("<--\t%s() rc %llu\n", __FUNCTION__, filesize); return filesize; } #endif @@ -4093,6 +4182,7 @@ */ static int FIO_decompressSrcFile(dRess_t ress, const char * dstfn, const char * srcfn) { +SPEW("-->\t%s(%p, %s, %s)\n", __FUNCTION__, &ress, dstfn, srcfn); FILE * srcfp; unsigned readSomething = 0; unsigned long long filesize = 0; @@ -4186,6 +4276,7 @@ if (remove(srcfn)) EXM_THROW(34, "zstd: %s: %s", srcfn, strerror(errno)); } +SPEW("<--\t%s(%p, %s, %s) rc %d\n", __FUNCTION__, &ress, dstfn, srcfn, 0); return 0; } @@ -4198,6 +4289,7 @@ static int FIO_decompressDstFile(dRess_t ress, const char * dstfn, const char * srcfn) { +SPEW("-->\t%s(%p, %s, %s)\n", __FUNCTION__, &ress, dstfn, srcfn); int result; stat_t sb; int stat_result = 0; @@ -4218,6 +4310,7 @@ result=1; /* don't do anything special if remove() fails */ else if (strcmp (dstfn, stdoutmark) && stat_result) UTIL_setFileStat(dstfn, &sb); +SPEW("<--\t%s(%p, %s, %s) rc %d\n", __FUNCTION__, &ress, dstfn, srcfn, result); return result; } @@ -4225,18 +4318,20 @@ int FIO_decompressFilename(const char * dstfn, const char * srcfn, const char * dfn) { +SPEW("-->\t%s(%s, %s, %s)\n", __FUNCTION__, dstfn, srcfn, dfn); int missingFiles = 0; dRess_t ress = FIO_createDResources(dfn); missingFiles += FIO_decompressDstFile(ress, dstfn, srcfn); FIO_freeDResources(ress); +SPEW("<--\t%s(%s, %s, %s) missing_files %d\n", __FUNCTION__, dstfn, srcfn, dfn, missingFiles); return missingFiles; } #define MAXSUFFIXSIZE 8 -int FIO_decompressMultipleFilenames(const char ** av, unsigned ac, +int FIO_decompressMultipleFilenames(ARGV_t av, int ac, const char * suffix, const char * dfn) { @@ -4244,14 +4339,15 @@ int missingFiles = 0; dRess_t ress = FIO_createDResources(dfn); +SPEW("-->\t%s(%p[%u], %s, %s)\n", __FUNCTION__, av, ac, suffix, dfn); if (suffix == NULL) EXM_THROW(70, "zstd: decompression: unknown dst"); /* should never happen */ if (!strcmp(suffix, stdoutmark) || !strcmp(suffix, nulmark)) { /* special cases : -c or -t */ ress.dstfp = FIO_openDstFile(suffix); if (ress.dstfp == 0) EXM_THROW(71, "cannot open %s", suffix); - for (unsigned u = 0; u < ac; u++) - missingFiles += FIO_decompressSrcFile(ress, suffix, av[u]); + for (int i = 0; i < ac; i++) + missingFiles += FIO_decompressSrcFile(ress, suffix, av[i]); if (fclose(ress.dstfp)) EXM_THROW(72, "Write error : cannot properly close stdout"); } else { @@ -4260,8 +4356,8 @@ char * dstfn = (char *)malloc(FNSPACE); if (dstfn == NULL) EXM_THROW(73, "not enough memory for dstfn"); - for (unsigned u = 0; u < ac; u++) { /* create dstfn */ - const char * const srcfn = av[u]; + for (int i = 0; i < ac; i++) { /* create dstfn */ + const char * const srcfn = av[i]; const char * const suffixPtr = strrchr(srcfn, '.'); size_t const sfnSize = strlen(srcfn); if (!suffixPtr) { @@ -4298,6 +4394,7 @@ } FIO_freeDResources(ress); +SPEW("<--\t%s(%p[%u], %s, %s) rc %d\n", __FUNCTION__, av, ac, suffix, dfn, missingFiles + skippedFiles); return missingFiles + skippedFiles; } @@ -4362,6 +4459,7 @@ **************************************/ static int usage(const char * programName) { +SPEW("-->\t%s()\n", __FUNCTION__); DISPLAY( "Usage :\n"); DISPLAY( " %s [args] [FILE(s)] [-o file]\n", programName); DISPLAY( "\n"); @@ -4380,6 +4478,7 @@ DISPLAY( "--rm : remove source file(s) after successful de/compression \n"); DISPLAY( " -k : preserve source file(s) (default) \n"); DISPLAY( " -h/-H : display help/long help and exit\n"); +SPEW("<--\t%s() rc %d\n", __FUNCTION__, 0); return 0; } @@ -4444,18 +4543,22 @@ DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n"); DISPLAY( "--priority=rt : set process priority to real-time\n"); #endif +SPEW("<--\t%s() rc %d\n", __FUNCTION__, 0); return 0; } static int badusage(const char * programName) { +SPEW("-->\t%s()\n", __FUNCTION__); DISPLAYLEVEL(1, "Incorrect parameters\n"); if (g_displayLevel >= 2) usage(programName); +SPEW("<--\t%s() rc %d\n", __FUNCTION__, 1); return 1; } static void waitEnter(void) { +SPEW("-->\t%s()\n", __FUNCTION__); int unused; DISPLAY("Press enter to continue...\n"); unused = getchar(); @@ -4464,9 +4567,11 @@ static const char * lastNameFromPath(const char * path) { +SPEW("-->\t%s()\n", __FUNCTION__); const char * name = path; if (strrchr(name, '/')) name = strrchr(name, '/') + 1; if (strrchr(name, '\\')) name = strrchr(name, '\\') + 1; /* windows */ +SPEW("<--\t%s(%s) name %s\n", __FUNCTION__, path, name); return name; } @@ -4475,8 +4580,11 @@ */ static int exeNameMatch(const char * exeName, const char * test) { - return !strncmp(exeName, test, strlen(test)) && +SPEW("-->\t%s()\n", __FUNCTION__); + int rc = !strncmp(exeName, test, strlen(test)) && (exeName[strlen(test)] == '\0' || exeName[strlen(test)] == '.'); +SPEW("<--\t%s(%s, %s) rc %d\n", __FUNCTION__, exeName, test, rc); + return rc; } /*! readU32FromChar() : @@ -4486,17 +4594,19 @@ Note : function result can overflow if digit string > MAX_UINT */ static unsigned readU32FromChar(const char ** sp) { - unsigned result = 0; +SPEW("-->\t%s()\n", __FUNCTION__); + unsigned rc = 0; while ((**sp >='0') && (**sp <='9')) - result *= 10, result += **sp - '0', (*sp)++ ; + rc *= 10, rc += **sp - '0', (*sp)++ ; if ((**sp == 'K') || (**sp == 'M')) { - result <<= 10; - if (**sp == 'M') result <<= 10; + rc <<= 10; + if (**sp == 'M') rc <<= 10; (*sp)++ ; if (**sp == 'i') (*sp)++; if (**sp == 'B') (*sp)++; } - return result; +SPEW("<--\t%s() rc %u\n", __FUNCTION__, rc); + return rc; } /** longCommandWArg() : @@ -4506,10 +4616,12 @@ */ static unsigned longCommandWArg(const char ** sp, const char * longCommand) { +SPEW("-->\t%s()\n", __FUNCTION__); size_t const comSize = strlen(longCommand); - int const result = !strncmp(*sp, longCommand, comSize); - if (result) *sp += comSize; - return result; + int const rc = !strncmp(*sp, longCommand, comSize); + if (rc) *sp += comSize; +SPEW("<--\t%s() rc %u\n", __FUNCTION__, rc); + return rc; } @@ -4522,6 +4634,7 @@ */ static unsigned parseCoverParameters(const char * s, COVER_params_t* params) { +SPEW("-->\t%s()\n", __FUNCTION__); memset(params, 0, sizeof(*params)); for (;;) { if (longCommandWArg(&s, "k=")) { params->k = readU32FromChar(&s); if (s[0] == ',') { s++; continue; } else break; } @@ -4531,6 +4644,7 @@ } if (s[0] != 0) return 0; DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nsteps=%u\n", params->k, params->d, params->steps); +SPEW("<--\t%s() rc %u\n", __FUNCTION__, 1); return 1; } @@ -4542,10 +4656,12 @@ */ static unsigned parseLegacyParameters(const char * s, unsigned* selectivity) { +SPEW("-->\t%s()\n", __FUNCTION__); if (!longCommandWArg(&s, "s=") && !longCommandWArg(&s, "selectivity=")) { return 0; } *selectivity = readU32FromChar(&s); if (s[0] != 0) return 0; DISPLAYLEVEL(4, "legacy: selectivity=%u\n", *selectivity); +SPEW("<--\t%s() rc %u\n", __FUNCTION__, 1); return 1; } @@ -4567,6 +4683,7 @@ */ static unsigned parseCompressionParameters(const char * s, ZSTD_compressionParameters* params) { +SPEW("-->\t%s()\n", __FUNCTION__); for (;;) { if (longCommandWArg(&s, "windowLog=") || longCommandWArg(&s, "wlog=")) { params->windowLog = readU32FromChar(&s); if (s[0] == ',') { s++; continue; } else break; } if (longCommandWArg(&s, "chainLog=") || longCommandWArg(&s, "clog=")) { params->chainLog = readU32FromChar(&s); if (s[0] == ',') { s++; continue; } else break; } @@ -4582,6 +4699,7 @@ if (s[0] != 0) return 0; /* check the end of string */ DISPLAYLEVEL(4, "windowLog=%d\nchainLog=%d\nhashLog=%d\nsearchLog=%d\n", params->windowLog, params->chainLog, params->hashLog, params->searchLog); DISPLAYLEVEL(4, "searchLength=%d\ntargetLength=%d\nstrategy=%d\n", params->searchLength, params->targetLength, params->strategy); +SPEW("<--\t%s() rc %u\n", __FUNCTION__, 1); return 1; } @@ -4591,10 +4709,6 @@ #define CLEAN_RETURN(i) { ec = (i); goto _end; } /*==============================================================*/ -#if defined(POPTIO_OPTION_PROCESSING) -static int _debug; -#endif /* POPTIO_OPTION_PROCESSING */ - int main(int argc, char *argv[]) { int forceStdout = 0, @@ -4618,7 +4732,7 @@ int cLevelLast = 1; unsigned recursive = 0; unsigned memLimit = 0; - const char ** av = (const char **)malloc(argc * sizeof(const char *)); /* argc >= 1 */ + ARGV_t av = (ARGV_t) malloc(argc * sizeof(const char *)); /* argc >= 1 */ unsigned ac = 0; const char * programName = argv[0]; const char * ofn = NULL; @@ -4923,6 +5037,8 @@ /* add filename to list */ av[ac++] = arg; } + av[ac] = NULL; +argvPrint(__FUNCTION__, av, NULL); if (lastCommand) { DISPLAY("error : command must be followed by arg \n"); CLEAN_RETURN(1); } /* forgotten arg */ @@ -4938,6 +5054,7 @@ DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION); #endif +SPEW("*** %s:%u nbThreads %d\n", __FUNCTION__, __LINE__, nbThreads); if (nbThreads == 0) { /* try to guess */ nbThreads = UTIL_countPhysicalCores(); @@ -4945,6 +5062,7 @@ } g_utilDisplayLevel = g_displayLevel; +SPEW("*** %s:%u followLinks %d\n", __FUNCTION__, __LINE__, followLinks); if (!followLinks) { nac = 0; for (unsigned u = 0; u < ac; u++) { @@ -4956,8 +5074,8 @@ } ac = nac; } - #ifdef UTIL_HAS_CREATEFILELIST +SPEW("*** %s:%u recursive %d\n", __FUNCTION__, __LINE__, recursive); if (recursive) { /* at this stage, av is a list of paths, which can contain both files and directories */ nav = UTIL_createFileList(av, ac, &navbuf, &nac, followLinks); if (nav) { @@ -4971,6 +5089,7 @@ #endif /* Check if benchmark is selected */ +SPEW("*** %s:%u op %d zom_bench %d\n", __FUNCTION__, __LINE__, operation, zom_bench); if (operation == zom_bench) { #ifndef ZSTD_NOBENCH BMK_setNotificationLevel(g_displayLevel); @@ -4984,8 +5103,8 @@ goto _end; } - /* Check if dictionary builder is selected */ +SPEW("*** %s:%u op %d zom_train %d\n", __FUNCTION__, __LINE__, operation, zom_train); if (operation == zom_train) { #ifndef ZSTD_NODICT if (cover) { @@ -5005,11 +5124,11 @@ ec = DiB_trainFromFiles(ofn, maxDictSize, av, ac, &dictParams, NULL, 0); } #endif - } goto _end; + } - /* No input filename ==> use stdin and stdout */ +SPEW("*** %s:%u op %d\n", __FUNCTION__, __LINE__, operation); ac += (ac == 0); /* av[0] is stdin by default */ if (!strcmp(av[0], stdinmark) && !ofn) ofn = stdoutmark; /* when input is stdin, default output is stdout */ @@ -5019,29 +5138,33 @@ CLEAN_RETURN(badusage(programName)); /* user-selected output filename, only possible with a single file */ +SPEW("*** %s:%u ofn %s stdoutmark %s nulmark %s\n", __FUNCTION__, __LINE__, ofn, stdoutmark, nulmark); if (ofn && strcmp(ofn, stdoutmark) && strcmp(ofn, nulmark) && (ac > 1)) { DISPLAY("Too many files (%u) on the command line. \n", ac); CLEAN_RETURN(ac); } - #ifndef ZSTD_NOCOMPRESS /* check compression level limits */ - int const maxCLevel = ultra ? ZSTD_maxCLevel() : ZSTDCLI_CLEVEL_MAX; - if (cLevel > maxCLevel) { - DISPLAYLEVEL(2, "Warning : compression level higher than max, reduced to %i \n", maxCLevel); - cLevel = maxCLevel; - } + int const maxCLevel = ultra ? ZSTD_maxCLevel() : ZSTDCLI_CLEVEL_MAX; +SPEW("*** %s:%u cLevel %d maxCLevel %d\n", __FUNCTION__, __LINE__, cLevel, maxCLevel); + if (cLevel > maxCLevel) { + DISPLAYLEVEL(2, "Warning : compression level higher than max, reduced to %i \n", maxCLevel); + cLevel = maxCLevel; + } #endif /* No status message in pipe mode (stdin - stdout) or multi-files mode */ +SPEW("*** %s:%u op %d\n", __FUNCTION__, __LINE__, operation); if (!strcmp(av[0], stdinmark) && ofn && !strcmp(ofn, stdoutmark) && (g_displayLevel == 2)) g_displayLevel=1; if ((ac >1 ) & (g_displayLevel == 2)) g_displayLevel = 1; /* IO Stream/File */ +SPEW("*** %s:%u op %d\n", __FUNCTION__, __LINE__, operation); FIO_setNotificationLevel(g_displayLevel); if (operation == zom_compress) { #ifndef ZSTD_NOCOMPRESS +SPEW("*** %s:%u op %d zom_compress %d\n", __FUNCTION__, __LINE__, operation, zom_compress); FIO_setNbThreads(nbThreads); FIO_setBlockSize((U32)blockSize); if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(g_overlapLog); @@ -5053,6 +5176,7 @@ DISPLAY("Compression not supported\n"); #endif } else { /* decompression or test */ +SPEW("*** %s:%u op %d\n", __FUNCTION__, __LINE__, operation); #ifndef ZSTD_NODECOMPRESS if (operation == zom_test) { ofn = nulmark; FIO_setRemoveSrcFile(0); } /* test mode */ FIO_setMemLimit(memLimit); @@ -5066,6 +5190,7 @@ } _end: +SPEW("*** %s:%u op %d\n", __FUNCTION__, __LINE__, operation); if (main_pause) waitEnter(); #ifdef UTIL_HAS_CREATEFILELIST if (nav) @@ -5080,4 +5205,3 @@ return ec; } - @@ . ______________________________________________________________________ RPM Package Manager http://rpm5.org CVS Sources Repository [email protected]
