Add local file scan and fix to url validator
Project: http://git-wip-us.apache.org/repos/asf/incubator-weex-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-weex-site/commit/8fcc5138 Tree: http://git-wip-us.apache.org/repos/asf/incubator-weex-site/tree/8fcc5138 Diff: http://git-wip-us.apache.org/repos/asf/incubator-weex-site/diff/8fcc5138 Branch: refs/heads/master Commit: 8fcc513871610090849dcaf4c3339b14c1d8624f Parents: ff1c81f Author: Zhenfei You <he...@imyzf.com> Authored: Thu Mar 22 17:05:51 2018 +0800 Committer: Zhenfei You <he...@imyzf.com> Committed: Thu Mar 22 17:06:26 2018 +0800 ---------------------------------------------------------------------- package-lock.json | 9 ++++++ package.json | 1 + test/url-validator.js | 69 ++++++++++++++++++++++++++++++++++++---------- 3 files changed, 65 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-weex-site/blob/8fcc5138/package-lock.json ---------------------------------------------------------------------- diff --git a/package-lock.json b/package-lock.json index c9344bd..660f566 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6130,6 +6130,15 @@ "limiter": "1.1.2" } }, + "string-similarity": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/string-similarity/-/string-similarity-1.2.0.tgz", + "integrity": "sha1-11FTyzg4RjGLejmo2SkrtNtOnDA=", + "dev": true, + "requires": { + "lodash": "4.17.4" + } + }, "string-width": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", http://git-wip-us.apache.org/repos/asf/incubator-weex-site/blob/8fcc5138/package.json ---------------------------------------------------------------------- diff --git a/package.json b/package.json index e2ba441..f567920 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "find-line-column": "^0.5.2", "glob": "^7.1.2", "node-fetch": "^2.1.1", + "string-similarity": "^1.2.0", "url-extractor": "^2.0.2" } } http://git-wip-us.apache.org/repos/asf/incubator-weex-site/blob/8fcc5138/test/url-validator.js ---------------------------------------------------------------------- diff --git a/test/url-validator.js b/test/url-validator.js index f020d52..198dc9b 100644 --- a/test/url-validator.js +++ b/test/url-validator.js @@ -1,8 +1,10 @@ -const glob = require('glob') const fs = require('fs') +const path = require('path') +const glob = require('glob') +const chalk = require('chalk') const fetch = require('node-fetch') const urlExtractor = require('url-extractor') -const chalk = require('chalk') +const similarity = require('string-similarity') const findLineColumn = require('find-line-column') const extractUrls = urlExtractor.extractUrls @@ -32,25 +34,64 @@ glob('source/**/*.md', (err, files) => { // '//xxx.com' -> 'http://xxx.com' if (url.match(/^\/\//)) url = 'http:' + url - // ignore non-http(s) - if (!url.match(/^https?:\/\//)) return // ignore whitelist for (let i = 0; i < whiteList.length; i++) { if (url.match(whiteList[i])) return } - // using fetch to test whether available - fetch(url) - .then(res => { - if (res.status >= 400) { - log(res.status, url, file, position); - } - }) - .catch(err => { - if (err.code !== 'ECONNREFUSED' && err.code !== 'ECONNRESET') { - log(err.code, url, file, position) + if (url.match(/^https?:\/\//)) { + // using fetch to test whether available + fetch(url) + .then(res => { + if (res.status >= 400) { + log(res.status, url, file, position); + } + }) + .catch(err => { + if (err.code !== 'ECONNREFUSED' && err.code !== 'ECONNRESET') { + log(err.code, url, file, position) + } + }) + } else if (url.match(/^.{0,2}\//)) { + // test local path + let targetPath + if (url.startsWith('/')) { + // '/wiki' -> 'source/wiki' + targetPath = path.resolve('source', url.substring(1)) + } else { + targetPath = path.resolve(path.dirname(file), url) + } + // 'a.html#title1' -> '#title1 + const urlHash = url.split('#')[1] ? '#' + url.split('#')[1] : '' + // 'a.html#title1' -> 'a.html' -> 'a.md + targetPath = targetPath.replace(/#(.*)/, '').replace('.html', '.md') + fs.access(targetPath, err => { + if (err) { + let patternDir + const option = { + cwd: 'source' + } + if (file.indexOf('/cn/') >= 0) { + patternDir = 'cn/**/' + } else { + patternDir = '**/' + option.ignore = 'cn/**' + } + glob(patternDir + path.basename(targetPath), option, (globError, files) => { + log(err.code, url, file, position) + if (files.length) { + console.log(chalk.bgBlue('FOUND'), files) + const match = similarity.findBestMatch(url, files) + .bestMatch.target.replace('.md', '.html') + console.log(chalk.bgGreen('FIXED'), match) + text = text.replace(url, `/${match}${urlHash}`) + // using sync mehtod to avoid writing a file in the same time. + fs.writeFileSync(file, text) + } + }) } }) + } }) }) })