Legoktm has uploaded a new change for review. https://gerrit.wikimedia.org/r/192619
Change subject: Make it easier to run scripts, remove Makefile ...................................................................... Make it easier to run scripts, remove Makefile * Add benchmark.php to "composer test" * Add "composer generate" command to run generate.php ** Now takes a --fetch parameter to re-download data files from the internet Bug: T90277 Change-Id: Ibf37f4dbf58d1e9227fb48623d2d64a035ad41d2 --- D Makefile M composer.json M scripts/generate.php 3 files changed, 46 insertions(+), 95 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/utfnormal refs/changes/19/192619/1 diff --git a/Makefile b/Makefile deleted file mode 100644 index b00154d..0000000 --- a/Makefile +++ /dev/null @@ -1,72 +0,0 @@ -.PHONY : all test testutf8 testclean icutest bench icubench clean distclean - -## Latest greatest version of Unicode -## May cause confusion if running test suite from these files -## when the data was generated from a previous version. -#BASE=http://www.unicode.org/Public/UNIDATA - -# Explicitly using Unicode 6.0 -BASE=http://www.unicode.org/Public/6.0.0/ucd - -# Can override to php-cli or php5 or whatever -PHP=php -#PHP=php-cli - -# Some nice tool to grab URLs with -FETCH=wget -#FETCH=fetch - -all : UtfNormalData.inc - -UtfNormalData.inc : UtfNormalGenerate.php UtfNormalUtil.php UnicodeData.txt CompositionExclusions.txt NormalizationCorrections.txt DerivedNormalizationProps.txt - $(PHP) UtfNormalGenerate.php - -test : UtfNormalTest.php UtfNormalData.inc NormalizationTest.txt - $(PHP) UtfNormalTest.php - -bench : UtfNormalData.inc testdata/washington.txt testdata/berlin.txt testdata/tokyo.txt testdata/young.txt testdata/bulgakov.txt - $(PHP) UtfNormalBench.php - -icutest : UtfNormalData.inc NormalizationTest.txt - $(PHP) Utf8Test.php --icu - $(PHP) UtfNormalTest.php --icu - -icubench : UtfNormalData.inc testdata/washington.txt testdata/berlin.txt testdata/tokyo.txt testdata/young.txt testdata/bulgakov.txt - $(PHP) UtfNormalBench.php --icu - -clean : - rm -f UtfNormalData.inc UtfNormalDataK.inc - -distclean : clean - rm -f CompositionExclusions.txt NormalizationTest.txt NormalizationCorrections.txt UnicodeData.txt DerivedNormalizationProps.txt UTF-8-test.txt - -# The Unicode data files... -CompositionExclusions.txt : - $(FETCH) $(BASE)/CompositionExclusions.txt - -NormalizationTest.txt : - $(FETCH) $(BASE)/NormalizationTest.txt - -NormalizationCorrections.txt : - $(FETCH) $(BASE)/NormalizationCorrections.txt - -DerivedNormalizationProps.txt : - $(FETCH) $(BASE)/DerivedNormalizationProps.txt - -UnicodeData.txt : - $(FETCH) $(BASE)/UnicodeData.txt - -testdata/berlin.txt : - mkdir -p testdata && wget -U MediaWiki/test -O testdata/berlin.txt "http://de.wikipedia.org/w/index.php?title=Berlin&oldid=2775712&action=raw"; - -testdata/washington.txt : - mkdir -p testdata && wget -U MediaWiki/test -O testdata/washington.txt "http://en.wikipedia.org/w/index.php?title=Washington%2C_D.C.&oldid=6370218&action=raw"; - -testdata/tokyo.txt : - mkdir -p testdata && wget -U MediaWiki/test -O testdata/tokyo.txt "http://ja.wikipedia.org/w/index.php?title=%E6%9D%B1%E4%BA%AC%E9%83%BD&oldid=940880&action=raw"; - -testdata/young.txt : - mkdir -p testdata && wget -U MediaWiki/test -O testdata/young.txt "http://ko.wikipedia.org/w/index.php?title=%EC%9D%B4%EC%88%98%EC%98%81&oldid=627688&action=raw"; - -testdata/bulgakov.txt : - mkdir -p testdata && wget -U MediaWiki/test -O testdata/bulgakov.txt "http://ru.wikipedia.org/w/index.php?title=%D0%91%D1%83%D0%BB%D0%B3%D0%B0%D0%BA%D0%BE%D0%B2%2C_%D0%A1%D0%B5%D1%80%D0%B3%D0%B5%D0%B9_%D0%9D%D0%B8%D0%BA%D0%BE%D0%BB%D0%B0%D0%B5%D0%B2%D0%B8%D1%87&oldid=17704&action=raw"; diff --git a/composer.json b/composer.json index 03208ef..bec748b 100644 --- a/composer.json +++ b/composer.json @@ -23,9 +23,11 @@ "ext-mbstring": "*" }, "scripts": { + "generate": "php scripts/generate.php", "test": [ "parallel-lint . --exclude vendor", - "phpunit $PHPUNIT_ARGS" + "phpunit $PHPUNIT_ARGS", + "php scripts/benchmark.php" ], "phpcs": "phpcs $PHPCS_ARGS --standard=vendor/mediawiki/mediawiki-codesniffer/MediaWiki --ignore=vendor -v ." }, @@ -33,4 +35,3 @@ "optimize-autoloader": true } } - diff --git a/scripts/generate.php b/scripts/generate.php index b95224a..117316b 100644 --- a/scripts/generate.php +++ b/scripts/generate.php @@ -33,13 +33,42 @@ require_once dirname( __DIR__ ) . '/vendor/autoload.php'; -$in = fopen( __DIR__ . "/data/DerivedNormalizationProps.txt", "rt" ); -if ( !$in ) { - print "Can't open DerivedNormalizationProps.txt for reading.\n"; - print "If necessary, fetch this file from the internet:\n"; - print "http://www.unicode.org/Public/UNIDATA/DerivedNormalizationProps.txt\n";; - exit( -1 ); +function download( $file, $url ) { + print "Downloading data from $url...\n"; + $fp = fopen( $file, 'w+' ); + $ch = curl_init( $url ); + curl_setopt( $ch, CURLOPT_FILE, $fp ); + curl_exec( $ch ); + curl_close( $ch ); + fclose( $fp ); } + +function getFilePointer( $file, $url ) { + if ( in_array( '--fetch', $_SERVER['argv'] ) ) { + download( $file, $url ); + } elseif ( !file_exists( $file ) ) { + print "Can't open $file for reading.\n"; + print "If necessary, fetch this file from the internet:\n"; + print "$url\n"; + print "Or re-run this script with --fetch\n"; + exit( -1 ); + } + + $fp = fopen( $file, "rt" ); + if ( !$fp ) { + // Eh? + print "Can't open $file for reading.\n"; + exit( -1 ); + } + + return $fp; +} + + +$in = getFilePointer( + __DIR__ . "/data/DerivedNormalizationProps.txt", + 'http://www.unicode.org/Public/UNIDATA/DerivedNormalizationProps.txt' +); print "Initializing normalization quick check tables...\n"; $checkNFC = array(); while ( false !== ( $line = fgets( $in ) ) ) { @@ -64,13 +93,10 @@ } fclose( $in ); -$in = fopen( __DIR__ . "/data/CompositionExclusions.txt", "rt" ); -if ( !$in ) { - print "Can't open CompositionExclusions.txt for reading.\n"; - print "If necessary, fetch this file from the internet:\n"; - print "http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt\n";; - exit( -1 ); -} +$in = getFilePointer( + __DIR__ . "/data/CompositionExclusions.txt", + 'http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt' +); $exclude = array(); while ( false !== ( $line = fgets( $in ) ) ) { if ( preg_match( '/^([0-9A-F]+)/i', $line, $matches ) ) { @@ -81,14 +107,10 @@ } fclose( $in ); -$in = fopen( __DIR__ . "/data/UnicodeData.txt", "rt" ); -if ( !$in ) { - print "Can't open UnicodeData.txt for reading.\n"; - print "If necessary, fetch this file from the internet:\n"; - print "http://www.unicode.org/Public/UNIDATA/UnicodeData.txt\n";; - exit( -1 ); -} - +$in = getFilePointer( + __DIR__ . "/data/UnicodeData.txt", + 'http://www.unicode.org/Public/UNIDATA/UnicodeData.txt' +); $compatibilityDecomp = array(); $canonicalDecomp = array(); $canonicalComp = array(); -- To view, visit https://gerrit.wikimedia.org/r/192619 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ibf37f4dbf58d1e9227fb48623d2d64a035ad41d2 Gerrit-PatchSet: 1 Gerrit-Project: utfnormal Gerrit-Branch: master Gerrit-Owner: Legoktm <legoktm.wikipe...@gmail.com> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
