From a4ef262b01d0b31b0caf2aeac32422bc80dc9a57 Mon Sep 17 00:00:00 2001 From: Sergey Melyukov Date: Tue, 20 Jun 2017 18:35:56 +0300 Subject: [PATCH 01/13] extract/l10n/Token: now ref could be explicit or implicit --- lib/extract/l10n/Token.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/extract/l10n/Token.js b/lib/extract/l10n/Token.js index 15d2dbb..9abd69f 100644 --- a/lib/extract/l10n/Token.js +++ b/lib/extract/l10n/Token.js @@ -10,14 +10,16 @@ var Token = function(dictionary, name){ Token.prototype.type = 'default'; Token.prototype.comment = null; -Token.prototype.addRef = function(file, refToken){ +Token.prototype.addRef = function(file, refToken, explicit){ for (var i = 0, ref; ref = this.ref[i]; i++) if (ref.file === file && ref.refToken === refToken) return; + this.hasExplicitRef = Boolean(this.hasExplicitRef || explicit); this.ref.push({ file: file, - refToken: refToken + refToken: refToken, + explicit: Boolean(explicit) }); }; From 8b1141b21aa571f48b5a59a9565a0dbe1ae3b197 Mon Sep 17 00:00:00 2001 From: Sergey Melyukov Date: Tue, 20 Jun 2017 18:36:39 +0300 Subject: [PATCH 02/13] common/files: added helper to files --- lib/common/files.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/common/files.js b/lib/common/files.js index 6b518bc..b4ce565 100644 --- a/lib/common/files.js +++ b/lib/common/files.js @@ -296,6 +296,10 @@ var FileManager = function(baseURI, relBaseURI, console, flow){ this.warns = []; this.readInfo = []; + + // helpers + this.unixpath = unixpath; + this.abspath = abspath; }; FileManager.prototype = { From c83cfbaeb5b0255c9b8477672faf4155ce63f847 Mon Sep 17 00:00:00 2001 From: Sergey Melyukov Date: Tue, 20 Jun 2017 18:37:13 +0300 Subject: [PATCH 03/13] extract/js: extract token refs from js --- lib/extract/js/l10n.js | 48 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/lib/extract/js/l10n.js b/lib/extract/js/l10n.js index b12dce4..dcb48e7 100644 --- a/lib/extract/js/l10n.js +++ b/lib/extract/js/l10n.js @@ -32,6 +32,34 @@ module.exports = function(file, flow, defineHandler, globalScope){ fconsole.log('[basis.l10n] basis.l10n.Dictionary#token.compute ' + id); token_.obj = resolveL10nToken(key + '.{?}', dictPath).jsToken.obj; + }), + token: at.createRunner(function(token_, this_, args, scope){ + var tokenKey = scope.simpleExpression(args[0]); + + if (tokenKey && tokenKey[0] == 'string') + { + tokenKey = tokenKey[1]; + + var file = tokenDescriptor.dictionary.file; + var id = key + '.' + tokenKey + '@' + file.filename; + // collect implicit refs + var parts = tokenKey.split('.'); + + fconsole.log('[basis.l10n] basis.l10n.Token#token ' + id); + parts.reduce(function(prev, current){ + var l10nToken = flow.l10n.getToken(key + '.' + prev + '@' + file.filename); + + l10nToken.addRef(this.file, token_); + + return prev + '.' + current; + }.bind(this)); + + // add explicit ref + var l10nToken = resolveL10nToken(key + '.' + tokenKey, file.filename); + + l10nToken.addRef(this.file, token_, true); + token_.obj = l10nToken.jsToken.obj; + } }) }; @@ -55,11 +83,25 @@ module.exports = function(file, flow, defineHandler, globalScope){ if (key && key[0] == 'string') { - var id = key[1] + '@' + file.filename; + key = key[1]; + + // collect implicit refs + var parts = key.split('.'); + var id = key + '@' + file.filename; + fconsole.log('[basis.l10n] basis.l10n.Dictionary#token ' + id); + parts.reduce(function(prev, current){ + var l10nToken = flow.l10n.getToken(prev + '@' + file.filename); + + l10nToken.addRef(this.file, token_); + + return prev + '.' + current; + }.bind(this)); + + // add explicit ref + var l10nToken = resolveL10nToken(key, file.filename); - var l10nToken = resolveL10nToken(key[1], file.filename); - l10nToken.addRef(this.file, token_); + l10nToken.addRef(this.file, token_, true); token_.obj = l10nToken.jsToken.obj; } else From 9cc2a4e3585b29039e0520d496cf8d11bb9041ff Mon Sep 17 00:00:00 2001 From: Sergey Melyukov Date: Tue, 20 Jun 2017 18:37:51 +0300 Subject: [PATCH 04/13] extract/js: extract token refs from templates --- lib/extract/l10n/index.js | 46 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/lib/extract/l10n/index.js b/lib/extract/l10n/index.js index 143c4b2..93743cb 100644 --- a/lib/extract/l10n/index.js +++ b/lib/extract/l10n/index.js @@ -25,6 +25,17 @@ var tmplAt = require('basisjs-tools-ast').tmpl; if (l10nPrefix.test(bindName)) { var l10nTokenRef = bindName.substr(5); + var parts = l10nTokenRef.split('@'); + var tokenName = parts[0]; + var dictFilename = parts[1]; + var tokenNameParts = tokenName.match(/^(.+?)\.{(.+?)}/); + + if (tokenNameParts && tokenNameParts.length == 3) { + tokenName = tokenNameParts[1]; + } + + l10nTokenRef = tokenName + '@' + dictFilename; + var l10nToken = flow.l10n.getToken(l10nTokenRef); var name = l10nToken.name; var dictionary = l10nToken.dictionary; @@ -43,7 +54,17 @@ var tmplAt = require('basisjs-tools-ast').tmpl; dictionary.file.jsRefCount++; dictionary.addRef(this.file); this.file.link(dictionary.file); - l10nToken.addRef(this.file, tmplRef); + + // collect implicit refs + tokenName.split('.').reduce(function(prev, current){ + var l10nToken = flow.l10n.getToken(prev + '@' + dictFilename); + + l10nToken.addRef(this.file, tmplRef); + + return prev + '.' + current; + }.bind(this)); + // add explicit ref + l10nToken.addRef(this.file, tmplRef, true); tmplRefs.push(tmplRef); } @@ -55,6 +76,17 @@ var tmplAt = require('basisjs-tools-ast').tmpl; if (l10nPrefix.test(bindName)) { var l10nTokenRef = bindName.substr(5); + var parts = l10nTokenRef.split('@'); + var tokenName = parts[0]; + var dictFilename = parts[1]; + var tokenNameParts = tokenName.match(/^(.+?)\.{(.+?)}/); + + if (tokenNameParts && tokenNameParts.length == 3) { + tokenName = tokenNameParts[1]; + } + + l10nTokenRef = tokenName + '@' + dictFilename; + var l10nToken = flow.l10n.getToken(l10nTokenRef); var name = l10nToken.name; var dictionary = l10nToken.dictionary; @@ -73,7 +105,17 @@ var tmplAt = require('basisjs-tools-ast').tmpl; dictionary.file.jsRefCount++; dictionary.addRef(this.file); this.file.link(dictionary.file); - l10nToken.addRef(this.file, tmplRef); + + // collect implicit refs + tokenName.split('.').reduce(function(prev, current){ + var l10nToken = flow.l10n.getToken(prev + '@' + dictFilename); + + l10nToken.addRef(this.file, tmplRef); + + return prev + '.' + current; + }.bind(this)); + // add explicit ref + l10nToken.addRef(this.file, tmplRef, true); tmplRefs.push(tmplRef); } From dc3762ca154c11e537ad0fd641f9bd1b4abcccd1 Mon Sep 17 00:00:00 2001 From: Sergey Melyukov Date: Tue, 20 Jun 2017 18:38:13 +0300 Subject: [PATCH 05/13] lint: collect unused l10n tokens --- lib/lint/collectUsed/l10n.js | 58 ++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 lib/lint/collectUsed/l10n.js diff --git a/lib/lint/collectUsed/l10n.js b/lib/lint/collectUsed/l10n.js new file mode 100644 index 0000000..4824e88 --- /dev/null +++ b/lib/lint/collectUsed/l10n.js @@ -0,0 +1,58 @@ +function isTarget(flow, basePath, collectPath, file){ + return file.filename && (basePath + file.filename).indexOf(collectPath + '/') === 0 && !flow.ignoreWarning(file.filename); +} + +module.exports = function collectUsedL10n(flow){ + var options = flow.options; + var basePath = options.base; + var collectPath = flow.files.abspath(basePath, options.warnUnusedL10n); + var usedTokens = {}; + + if (!flow.l10n || !flow.l10n.dictionaries) + return; + + for (var dictFilename in flow.l10n.dictionaries) + { + if (isTarget(flow, basePath, collectPath, { filename: dictFilename })) + { + var dict = flow.l10n.dictionaries[dictFilename]; + var dictTokens = dict.tokens; + + // mark tokens as explicitly, implicitly or not used + for (var tokenName in dictTokens) + { + var tokenInfo = dictTokens[tokenName]; + + usedTokens[dictFilename] = usedTokens[dictFilename] || {}; + + if (tokenInfo.ref.length) + usedTokens[dictFilename][tokenName] = tokenInfo.hasExplicitRef ? true : 'implicit'; + else + usedTokens[dictFilename][tokenName] = false; + } + + // mark unused tokens in used branches as implicitly used + for (tokenName in usedTokens[dictFilename]) + { + if (!usedTokens[dictFilename][tokenName]) + { + var lastDotIx = tokenName.lastIndexOf('.'); + + if (lastDotIx > -1) + { + var branchName = tokenName.slice(0, lastDotIx); + + if (usedTokens[dictFilename][branchName]) + usedTokens[dictFilename][tokenName] = 'implicit'; + } + } + } + } + } + + flow.usedL10nTokens = { + basePath: basePath, + collectPath: collectPath, + items: usedTokens + }; +}; From 89b1c1ee84d1ea283330ae5d510ccad4b18e9248 Mon Sep 17 00:00:00 2001 From: Sergey Melyukov Date: Tue, 20 Jun 2017 18:39:24 +0300 Subject: [PATCH 06/13] lint/reporter: handle unused l10n tokens in the reporters --- lib/lint/reporter/parallel-process-warns.js | 42 +++++++++++++++++++++ lib/lint/reporter/process-warns.js | 28 +++++++++++++- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/lib/lint/reporter/parallel-process-warns.js b/lib/lint/reporter/parallel-process-warns.js index 70f3309..02daa00 100644 --- a/lib/lint/reporter/parallel-process-warns.js +++ b/lib/lint/reporter/parallel-process-warns.js @@ -1,5 +1,6 @@ module.exports = function(tasks){ var result = {}; + var unusedL10nTokens = {}; tasks.forEach(function(task){ var failures = result[task.name] = []; @@ -20,7 +21,48 @@ module.exports = function(tasks){ message: warn.message + ' at ' + filename }); }); + + if (task.result.usedL10nTokens) + { + for (var dictFileName in task.result.usedL10nTokens.items) { + var absDictFilename = task.result.usedL10nTokens.basePath + dictFileName; + var tokenNames = task.result.usedL10nTokens.items[dictFileName]; + + unusedL10nTokens[absDictFilename] = unusedL10nTokens[absDictFilename] || {}; + + for (var tokenName in tokenNames) + { + if (tokenNames.hasOwnProperty(tokenName)) + { + if (!tokenNames[tokenName]) + unusedL10nTokens[absDictFilename][tokenName] = true; + else + delete unusedL10nTokens[absDictFilename][tokenName]; + } + } + + if (!Object.keys(unusedL10nTokens[absDictFilename]).length) + delete unusedL10nTokens[absDictFilename]; + } + } }); + if (Object.keys(unusedL10nTokens).length) { + for (var dictFileName in unusedL10nTokens){ + var tokenNames = unusedL10nTokens[dictFileName]; + + dictFileName = dictFileName.slice(process.cwd().length); + + for (var tokenName in tokenNames) + { + result['unused l10n tokens'] = result['unused l10n tokens'] || []; + result['unused l10n tokens'].push({ + loc: dictFileName, + message: tokenName + ' at ' + dictFileName + }); + } + } + } + return result; }; diff --git a/lib/lint/reporter/process-warns.js b/lib/lint/reporter/process-warns.js index 9c1ecd3..23fc4cd 100644 --- a/lib/lint/reporter/process-warns.js +++ b/lib/lint/reporter/process-warns.js @@ -1,5 +1,9 @@ -module.exports = function(warns, fileFilter){ +var isChildProcess = typeof process.send == 'function'; // child process has send method + +module.exports = function(flow, options){ var result = {}; + var warns = flow.warns; + var fileFilter = options.filter; warns.forEach(function(warn){ var filename = warn.file || ''; @@ -16,5 +20,27 @@ module.exports = function(warns, fileFilter){ }); }); + if (!isChildProcess) + { + if (options.warnUnusedL10n && flow.usedL10nTokens) + { + var usedL10nTokensInfo = flow.usedL10nTokens; + + for (var dictFileName in usedL10nTokensInfo.items){ + var tokenNames = usedL10nTokensInfo.items[dictFileName]; + + for (var tokenName in tokenNames) { + if (!tokenNames[tokenName]) { + result[dictFileName] = result[dictFileName] || []; + result[dictFileName].push({ + loc: dictFileName, + message: tokenName + }); + } + } + } + } + } + return result; }; From 99cd37b6e3b846149a132419155dbcbf14867d12 Mon Sep 17 00:00:00 2001 From: Sergey Melyukov Date: Tue, 20 Jun 2017 18:40:00 +0300 Subject: [PATCH 07/13] lint: added --warn-unused-l10n to linter parameters --- lib/lint/command.js | 1 + lib/lint/index.js | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/lint/command.js b/lib/lint/command.js index 37ed6a1..dcb7786 100644 --- a/lib/lint/command.js +++ b/lib/lint/command.js @@ -20,6 +20,7 @@ module.exports = clap.create('lint', '[fileOrPreset]') .option('--no-color', 'Suppress color output') .option('--silent', 'No any output') + .option('--warn-unused-l10n ', 'Warn about unused l10n tokens in specified path. Do not use with --js-cut-dev. It might cause incorrect result') .option('--filter ', 'Show warnings only for specified file', resolveCwd) .option('-r, --reporter ', 'Reporter console (default), checkstyle, junit', function(reporter){ diff --git a/lib/lint/index.js b/lib/lint/index.js index 8a73339..bcc2fff 100644 --- a/lib/lint/index.js +++ b/lib/lint/index.js @@ -7,6 +7,9 @@ var extract = require('../extract'); var command = require('./command'); var chalk = require('chalk'); var isChildProcess = typeof process.send == 'function'; // child process has send method +var collectUsed = { + l10n: require('./collectUsed/l10n') +}; if (isChildProcess) process.on('uncaughtException', function(error){ @@ -91,7 +94,11 @@ function lint(config){ l10nInfo: true }).concat([ function(flow){ - flow.result = require('./reporter/process-warns')(flow.warns, options.filter); + if (options.warnUnusedL10n) + collectUsed.l10n(flow); + }, + function(flow){ + flow.result = require('./reporter/process-warns')(flow, options); } ]); @@ -148,6 +155,7 @@ function lint(config){ event: 'done', success: !flow.warns.length, warnings: flow.warns, + usedL10nTokens: flow.usedL10nTokens, result: flow.result }); }); From fc40da01b9abcc17ab6fd2de6347f96d42518c1f Mon Sep 17 00:00:00 2001 From: Sergey Melyukov Date: Thu, 22 Jun 2017 18:58:11 +0300 Subject: [PATCH 08/13] lint/unused/l10n: improved search unused algorithm --- lib/lint/collectUsed/l10n.js | 58 ----------------------- lib/lint/unused/l10n.js | 92 ++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 58 deletions(-) delete mode 100644 lib/lint/collectUsed/l10n.js create mode 100644 lib/lint/unused/l10n.js diff --git a/lib/lint/collectUsed/l10n.js b/lib/lint/collectUsed/l10n.js deleted file mode 100644 index 4824e88..0000000 --- a/lib/lint/collectUsed/l10n.js +++ /dev/null @@ -1,58 +0,0 @@ -function isTarget(flow, basePath, collectPath, file){ - return file.filename && (basePath + file.filename).indexOf(collectPath + '/') === 0 && !flow.ignoreWarning(file.filename); -} - -module.exports = function collectUsedL10n(flow){ - var options = flow.options; - var basePath = options.base; - var collectPath = flow.files.abspath(basePath, options.warnUnusedL10n); - var usedTokens = {}; - - if (!flow.l10n || !flow.l10n.dictionaries) - return; - - for (var dictFilename in flow.l10n.dictionaries) - { - if (isTarget(flow, basePath, collectPath, { filename: dictFilename })) - { - var dict = flow.l10n.dictionaries[dictFilename]; - var dictTokens = dict.tokens; - - // mark tokens as explicitly, implicitly or not used - for (var tokenName in dictTokens) - { - var tokenInfo = dictTokens[tokenName]; - - usedTokens[dictFilename] = usedTokens[dictFilename] || {}; - - if (tokenInfo.ref.length) - usedTokens[dictFilename][tokenName] = tokenInfo.hasExplicitRef ? true : 'implicit'; - else - usedTokens[dictFilename][tokenName] = false; - } - - // mark unused tokens in used branches as implicitly used - for (tokenName in usedTokens[dictFilename]) - { - if (!usedTokens[dictFilename][tokenName]) - { - var lastDotIx = tokenName.lastIndexOf('.'); - - if (lastDotIx > -1) - { - var branchName = tokenName.slice(0, lastDotIx); - - if (usedTokens[dictFilename][branchName]) - usedTokens[dictFilename][tokenName] = 'implicit'; - } - } - } - } - } - - flow.usedL10nTokens = { - basePath: basePath, - collectPath: collectPath, - items: usedTokens - }; -}; diff --git a/lib/lint/unused/l10n.js b/lib/lint/unused/l10n.js new file mode 100644 index 0000000..15b298e --- /dev/null +++ b/lib/lint/unused/l10n.js @@ -0,0 +1,92 @@ +function isTarget(flow, basePath, collectPath, file){ + return file.filename && (basePath + file.filename).indexOf(collectPath + '/') === 0 && !flow.ignoreWarning(file.filename); +} + +exports.collectUsed = function(flow){ + var options = flow.options; + var basePath = options.base; + var collectPath = flow.files.abspath(basePath, options.warnUnusedL10n); + var usedTokens = {}; + + if (!flow.l10n || !flow.l10n.dictionaries) + return; + + for (var dictFilename in flow.l10n.dictionaries) + { + if (isTarget(flow, basePath, collectPath, { filename: dictFilename })) + { + var dict = flow.l10n.dictionaries[dictFilename]; + var dictTokens = dict.tokens; + + // mark tokens as explicitly, implicitly or not used + for (var tokenName in dictTokens) + { + var tokenInfo = dictTokens[tokenName]; + + usedTokens[dictFilename] = usedTokens[dictFilename] || {}; + + if (tokenInfo.ref.length) + usedTokens[dictFilename][tokenName] = tokenInfo.hasExplicitRef ? true : 'implicit'; + else + usedTokens[dictFilename][tokenName] = false; + } + + // mark unused tokens in explicitly used branches as implicitly used + for (tokenName in usedTokens[dictFilename]) + { + if (!usedTokens[dictFilename][tokenName]) + { + var parts = tokenName.split('.'); + var passed = []; + + if (parts.length > 1) + { + // collect branches till explicitly used branch + while (parts.length) + { + var currentPath = parts.join('.'); + + if (usedTokens[dictFilename][currentPath] === true) + break; + + passed.push(currentPath); + parts.pop(); + } + + // if we have an explicitly used branch then mark collected branches as implicitly used + if (parts.length) + for (var i = 0; i < passed.length; i++) + usedTokens[dictFilename][passed[i]] = 'implicit'; + } + } + } + } + } + + return { + basePath: basePath, + collectPath: collectPath, + items: usedTokens + }; +}; + +exports.warn = function(flow){ + if (flow.options.warnUnusedL10n && flow.usedL10nTokens) + { + var usedL10nTokensInfo = flow.usedL10nTokens; + + for (var dictFileName in usedL10nTokensInfo.items){ + var tokenNames = usedL10nTokensInfo.items[dictFileName]; + + for (var tokenName in tokenNames) { + if (!tokenNames[tokenName]) { + dictFileName = dictFileName.slice(process.cwd().length); + flow.warn({ + file: dictFileName, + message: tokenName + }); + } + } + } + } +}; From 3a5e38403110f518e18c922b13df16879dcd9d03 Mon Sep 17 00:00:00 2001 From: Sergey Melyukov Date: Thu, 22 Jun 2017 18:59:10 +0300 Subject: [PATCH 09/13] lint: improved parallel l10n reporter and minor fixes --- lib/lint/command.js | 5 +- lib/lint/index.js | 11 ++-- .../reporter/parallel-process-unused-l10n.js | 62 +++++++++++++++++++ lib/lint/reporter/parallel-process-warns.js | 44 +------------ 4 files changed, 74 insertions(+), 48 deletions(-) create mode 100644 lib/lint/reporter/parallel-process-unused-l10n.js diff --git a/lib/lint/command.js b/lib/lint/command.js index dcb7786..a7493dc 100644 --- a/lib/lint/command.js +++ b/lib/lint/command.js @@ -2,6 +2,7 @@ var path = require('path'); var clap = require.main.require('clap'); var common = require('../common/command'); var isChildProcess = typeof process.send == 'function'; // child process has send method +var handleUnusedL10n = require('./reporter/parallel-process-unused-l10n'); function resolveCwd(value){ return path.resolve(process.env.PWD || process.cwd(), value); @@ -20,7 +21,7 @@ module.exports = clap.create('lint', '[fileOrPreset]') .option('--no-color', 'Suppress color output') .option('--silent', 'No any output') - .option('--warn-unused-l10n ', 'Warn about unused l10n tokens in specified path. Do not use with --js-cut-dev. It might cause incorrect result') + .option('--warn-unused-l10n ', 'Warn about unused l10n tokens for specified path. Avoid using with --js-cut-dev since it might cause to incorrect results') .option('--filter ', 'Show warnings only for specified file', resolveCwd) .option('-r, --reporter ', 'Reporter console (default), checkstyle, junit', function(reporter){ @@ -54,6 +55,8 @@ module.exports.getParallelOptions = function(){ return { silent: true, callback: function(res){ + handleUnusedL10n(res); + var reporter = require(require('./reporter')[command.values.reporter]); var data = require('./reporter/parallel-process-warns.js')(res); console.log(reporter(data)); diff --git a/lib/lint/index.js b/lib/lint/index.js index bcc2fff..defc6a0 100644 --- a/lib/lint/index.js +++ b/lib/lint/index.js @@ -7,9 +7,7 @@ var extract = require('../extract'); var command = require('./command'); var chalk = require('chalk'); var isChildProcess = typeof process.send == 'function'; // child process has send method -var collectUsed = { - l10n: require('./collectUsed/l10n') -}; +var unusedL10n = require('./unused/l10n'); if (isChildProcess) process.on('uncaughtException', function(error){ @@ -95,7 +93,12 @@ function lint(config){ }).concat([ function(flow){ if (options.warnUnusedL10n) - collectUsed.l10n(flow); + { + flow.usedL10nTokens = unusedL10n.collectUsed(flow); + if (!isChildProcess) + unusedL10n.warn(flow); + } + }, function(flow){ flow.result = require('./reporter/process-warns')(flow, options); diff --git a/lib/lint/reporter/parallel-process-unused-l10n.js b/lib/lint/reporter/parallel-process-unused-l10n.js new file mode 100644 index 0000000..4b60c8f --- /dev/null +++ b/lib/lint/reporter/parallel-process-unused-l10n.js @@ -0,0 +1,62 @@ +module.exports = function handleUnusedL10n(tasks){ + var usedL10nTokens = {}; + var unusedL10nTokens = {}; + + // merge used l10n tokens from tasks + // and collect used and unused l10n tokens + tasks.forEach(function(task){ + if (task.result.usedL10nTokens) + for (var dictFileName in task.result.usedL10nTokens.items) + { + var absDictFilename = task.result.usedL10nTokens.basePath + dictFileName; + var tokenUsageInfo = task.result.usedL10nTokens.items[dictFileName]; + + usedL10nTokens[absDictFilename] = usedL10nTokens[absDictFilename] || {}; + unusedL10nTokens[absDictFilename] = unusedL10nTokens[absDictFilename] || {}; + + for (var tokenName in tokenUsageInfo) + if (tokenUsageInfo.hasOwnProperty(tokenName)) + { + var used = tokenUsageInfo[tokenName]; + var alreadyUsed = usedL10nTokens[absDictFilename].hasOwnProperty(tokenName); + + if (alreadyUsed) + continue; + + if (!used) + unusedL10nTokens[absDictFilename][tokenName] = true; + else + { + usedL10nTokens[absDictFilename][tokenName] = true; + delete unusedL10nTokens[absDictFilename][tokenName]; + } + } + + if (!Object.keys(unusedL10nTokens[absDictFilename]).length) + delete unusedL10nTokens[absDictFilename]; + } + }); + + // warn about unused l10n tokens + if (Object.keys(unusedL10nTokens).length) + { + for (var dictFileName in unusedL10nTokens) + { + var tokenNames = unusedL10nTokens[dictFileName]; + + dictFileName = dictFileName.slice(process.cwd().length); + + var key = 'unused l10n tokens at ' + dictFileName; + var task = { name: key, result: { warnings: [] } }; + + for (var tokenName in tokenNames) + { + task.result.warnings.push({ + file: tokenName + }); + } + + tasks.push(task); + } + } +}; diff --git a/lib/lint/reporter/parallel-process-warns.js b/lib/lint/reporter/parallel-process-warns.js index 02daa00..66bdc89 100644 --- a/lib/lint/reporter/parallel-process-warns.js +++ b/lib/lint/reporter/parallel-process-warns.js @@ -1,6 +1,5 @@ module.exports = function(tasks){ var result = {}; - var unusedL10nTokens = {}; tasks.forEach(function(task){ var failures = result[task.name] = []; @@ -18,51 +17,10 @@ module.exports = function(tasks){ failures.push({ loc: warn.loc, - message: warn.message + ' at ' + filename + message: warn.message ? warn.message + ' at ' + filename : filename }); }); - - if (task.result.usedL10nTokens) - { - for (var dictFileName in task.result.usedL10nTokens.items) { - var absDictFilename = task.result.usedL10nTokens.basePath + dictFileName; - var tokenNames = task.result.usedL10nTokens.items[dictFileName]; - - unusedL10nTokens[absDictFilename] = unusedL10nTokens[absDictFilename] || {}; - - for (var tokenName in tokenNames) - { - if (tokenNames.hasOwnProperty(tokenName)) - { - if (!tokenNames[tokenName]) - unusedL10nTokens[absDictFilename][tokenName] = true; - else - delete unusedL10nTokens[absDictFilename][tokenName]; - } - } - - if (!Object.keys(unusedL10nTokens[absDictFilename]).length) - delete unusedL10nTokens[absDictFilename]; - } - } }); - if (Object.keys(unusedL10nTokens).length) { - for (var dictFileName in unusedL10nTokens){ - var tokenNames = unusedL10nTokens[dictFileName]; - - dictFileName = dictFileName.slice(process.cwd().length); - - for (var tokenName in tokenNames) - { - result['unused l10n tokens'] = result['unused l10n tokens'] || []; - result['unused l10n tokens'].push({ - loc: dictFileName, - message: tokenName + ' at ' + dictFileName - }); - } - } - } - return result; }; From bdd24234bc46359bae8fc5d6fd6e2b9f34388fd6 Mon Sep 17 00:00:00 2001 From: Sergey Melyukov Date: Thu, 22 Jun 2017 19:06:49 +0300 Subject: [PATCH 10/13] common/files: removed unused helper --- lib/common/files.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/common/files.js b/lib/common/files.js index b4ce565..03b94cf 100644 --- a/lib/common/files.js +++ b/lib/common/files.js @@ -298,7 +298,6 @@ var FileManager = function(baseURI, relBaseURI, console, flow){ this.readInfo = []; // helpers - this.unixpath = unixpath; this.abspath = abspath; }; From 779caa6e914cd4393ed3ed8c6ddf1b60a107c8c5 Mon Sep 17 00:00:00 2001 From: Sergey Melyukov Date: Thu, 22 Jun 2017 19:10:48 +0300 Subject: [PATCH 11/13] lint: revert reporter --- lib/lint/index.js | 4 +--- lib/lint/reporter/process-warns.js | 28 +--------------------------- lib/lint/unused/l10n.js | 1 - 3 files changed, 2 insertions(+), 31 deletions(-) diff --git a/lib/lint/index.js b/lib/lint/index.js index defc6a0..2634d5a 100644 --- a/lib/lint/index.js +++ b/lib/lint/index.js @@ -99,9 +99,7 @@ function lint(config){ unusedL10n.warn(flow); } - }, - function(flow){ - flow.result = require('./reporter/process-warns')(flow, options); + flow.result = require('./reporter/process-warns')(flow.warns, options.filter); } ]); diff --git a/lib/lint/reporter/process-warns.js b/lib/lint/reporter/process-warns.js index 23fc4cd..9c1ecd3 100644 --- a/lib/lint/reporter/process-warns.js +++ b/lib/lint/reporter/process-warns.js @@ -1,9 +1,5 @@ -var isChildProcess = typeof process.send == 'function'; // child process has send method - -module.exports = function(flow, options){ +module.exports = function(warns, fileFilter){ var result = {}; - var warns = flow.warns; - var fileFilter = options.filter; warns.forEach(function(warn){ var filename = warn.file || ''; @@ -20,27 +16,5 @@ module.exports = function(flow, options){ }); }); - if (!isChildProcess) - { - if (options.warnUnusedL10n && flow.usedL10nTokens) - { - var usedL10nTokensInfo = flow.usedL10nTokens; - - for (var dictFileName in usedL10nTokensInfo.items){ - var tokenNames = usedL10nTokensInfo.items[dictFileName]; - - for (var tokenName in tokenNames) { - if (!tokenNames[tokenName]) { - result[dictFileName] = result[dictFileName] || []; - result[dictFileName].push({ - loc: dictFileName, - message: tokenName - }); - } - } - } - } - } - return result; }; diff --git a/lib/lint/unused/l10n.js b/lib/lint/unused/l10n.js index 15b298e..85da4e7 100644 --- a/lib/lint/unused/l10n.js +++ b/lib/lint/unused/l10n.js @@ -80,7 +80,6 @@ exports.warn = function(flow){ for (var tokenName in tokenNames) { if (!tokenNames[tokenName]) { - dictFileName = dictFileName.slice(process.cwd().length); flow.warn({ file: dictFileName, message: tokenName From 4ee3510e4d0096a37357af4d87222d2ae5fefc32 Mon Sep 17 00:00:00 2001 From: Sergey Melyukov Date: Fri, 23 Jun 2017 12:16:48 +0300 Subject: [PATCH 12/13] more accuracy l10n token usage mark --- lib/extract/js/l10n.js | 12 ++++++------ lib/extract/l10n/Token.js | 7 ++++--- lib/extract/l10n/index.js | 12 ++++++------ lib/lint/unused/l10n.js | 19 +++++++++++++++---- 4 files changed, 31 insertions(+), 19 deletions(-) diff --git a/lib/extract/js/l10n.js b/lib/extract/js/l10n.js index dcb48e7..2400995 100644 --- a/lib/extract/js/l10n.js +++ b/lib/extract/js/l10n.js @@ -42,14 +42,14 @@ module.exports = function(file, flow, defineHandler, globalScope){ var file = tokenDescriptor.dictionary.file; var id = key + '.' + tokenKey + '@' + file.filename; - // collect implicit refs + // collect through refs var parts = tokenKey.split('.'); fconsole.log('[basis.l10n] basis.l10n.Token#token ' + id); parts.reduce(function(prev, current){ var l10nToken = flow.l10n.getToken(key + '.' + prev + '@' + file.filename); - l10nToken.addRef(this.file, token_); + l10nToken.addRef(this.file, token_, 'through'); return prev + '.' + current; }.bind(this)); @@ -57,7 +57,7 @@ module.exports = function(file, flow, defineHandler, globalScope){ // add explicit ref var l10nToken = resolveL10nToken(key + '.' + tokenKey, file.filename); - l10nToken.addRef(this.file, token_, true); + l10nToken.addRef(this.file, token_, 'explicit'); token_.obj = l10nToken.jsToken.obj; } }) @@ -85,7 +85,7 @@ module.exports = function(file, flow, defineHandler, globalScope){ { key = key[1]; - // collect implicit refs + // collect through refs var parts = key.split('.'); var id = key + '@' + file.filename; @@ -93,7 +93,7 @@ module.exports = function(file, flow, defineHandler, globalScope){ parts.reduce(function(prev, current){ var l10nToken = flow.l10n.getToken(prev + '@' + file.filename); - l10nToken.addRef(this.file, token_); + l10nToken.addRef(this.file, token_, 'through'); return prev + '.' + current; }.bind(this)); @@ -101,7 +101,7 @@ module.exports = function(file, flow, defineHandler, globalScope){ // add explicit ref var l10nToken = resolveL10nToken(key, file.filename); - l10nToken.addRef(this.file, token_, true); + l10nToken.addRef(this.file, token_, 'explicit'); token_.obj = l10nToken.jsToken.obj; } else diff --git a/lib/extract/l10n/Token.js b/lib/extract/l10n/Token.js index 9abd69f..945efde 100644 --- a/lib/extract/l10n/Token.js +++ b/lib/extract/l10n/Token.js @@ -10,16 +10,17 @@ var Token = function(dictionary, name){ Token.prototype.type = 'default'; Token.prototype.comment = null; -Token.prototype.addRef = function(file, refToken, explicit){ +Token.prototype.addRef = function(file, refToken, type){ for (var i = 0, ref; ref = this.ref[i]; i++) if (ref.file === file && ref.refToken === refToken) return; - this.hasExplicitRef = Boolean(this.hasExplicitRef || explicit); + // explicit has more priority than through + this.usage = type === 'explicit' ? type : this.usage || type; this.ref.push({ file: file, refToken: refToken, - explicit: Boolean(explicit) + type: type }); }; diff --git a/lib/extract/l10n/index.js b/lib/extract/l10n/index.js index 93743cb..99a8660 100644 --- a/lib/extract/l10n/index.js +++ b/lib/extract/l10n/index.js @@ -55,16 +55,16 @@ var tmplAt = require('basisjs-tools-ast').tmpl; dictionary.addRef(this.file); this.file.link(dictionary.file); - // collect implicit refs + // collect through refs tokenName.split('.').reduce(function(prev, current){ var l10nToken = flow.l10n.getToken(prev + '@' + dictFilename); - l10nToken.addRef(this.file, tmplRef); + l10nToken.addRef(this.file, tmplRef, 'through'); return prev + '.' + current; }.bind(this)); // add explicit ref - l10nToken.addRef(this.file, tmplRef, true); + l10nToken.addRef(this.file, tmplRef, 'explicit'); tmplRefs.push(tmplRef); } @@ -106,16 +106,16 @@ var tmplAt = require('basisjs-tools-ast').tmpl; dictionary.addRef(this.file); this.file.link(dictionary.file); - // collect implicit refs + // collect through refs tokenName.split('.').reduce(function(prev, current){ var l10nToken = flow.l10n.getToken(prev + '@' + dictFilename); - l10nToken.addRef(this.file, tmplRef); + l10nToken.addRef(this.file, tmplRef, 'through'); return prev + '.' + current; }.bind(this)); // add explicit ref - l10nToken.addRef(this.file, tmplRef, true); + l10nToken.addRef(this.file, tmplRef, 'explicit'); tmplRefs.push(tmplRef); } diff --git a/lib/lint/unused/l10n.js b/lib/lint/unused/l10n.js index 85da4e7..0ff4211 100644 --- a/lib/lint/unused/l10n.js +++ b/lib/lint/unused/l10n.js @@ -2,6 +2,17 @@ function isTarget(flow, basePath, collectPath, file){ return file.filename && (basePath + file.filename).indexOf(collectPath + '/') === 0 && !flow.ignoreWarning(file.filename); } +var TOKEN_REF_UNUSED = 0; +var TOKEN_REF_THROUGH = 1; +var TOKEN_REF_IMPLICIT = 2; +var TOKEN_REF_EXPLICIT = 3; + +var usageMap = { + through: TOKEN_REF_THROUGH, + implicit: TOKEN_REF_IMPLICIT, + explicit: TOKEN_REF_EXPLICIT +}; + exports.collectUsed = function(flow){ var options = flow.options; var basePath = options.base; @@ -26,9 +37,9 @@ exports.collectUsed = function(flow){ usedTokens[dictFilename] = usedTokens[dictFilename] || {}; if (tokenInfo.ref.length) - usedTokens[dictFilename][tokenName] = tokenInfo.hasExplicitRef ? true : 'implicit'; + usedTokens[dictFilename][tokenName] = usageMap[tokenInfo.usage]; else - usedTokens[dictFilename][tokenName] = false; + usedTokens[dictFilename][tokenName] = TOKEN_REF_UNUSED; } // mark unused tokens in explicitly used branches as implicitly used @@ -46,7 +57,7 @@ exports.collectUsed = function(flow){ { var currentPath = parts.join('.'); - if (usedTokens[dictFilename][currentPath] === true) + if (usedTokens[dictFilename][currentPath] === TOKEN_REF_EXPLICIT) break; passed.push(currentPath); @@ -56,7 +67,7 @@ exports.collectUsed = function(flow){ // if we have an explicitly used branch then mark collected branches as implicitly used if (parts.length) for (var i = 0; i < passed.length; i++) - usedTokens[dictFilename][passed[i]] = 'implicit'; + usedTokens[dictFilename][passed[i]] = TOKEN_REF_IMPLICIT; } } } From cdf28c2845ad43f39150b2147c6ad37513073dbb Mon Sep 17 00:00:00 2001 From: Sergey Melyukov Date: Thu, 29 Jun 2017 11:05:45 +0300 Subject: [PATCH 13/13] lint: minor improvements to unused-l10n reporter --- .../reporter/parallel-process-unused-l10n.js | 24 +++++++++---------- lib/lint/unused/l10n.js | 2 +- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/lib/lint/reporter/parallel-process-unused-l10n.js b/lib/lint/reporter/parallel-process-unused-l10n.js index 4b60c8f..30d2403 100644 --- a/lib/lint/reporter/parallel-process-unused-l10n.js +++ b/lib/lint/reporter/parallel-process-unused-l10n.js @@ -1,3 +1,5 @@ +var path = require('path'); + module.exports = function handleUnusedL10n(tasks){ var usedL10nTokens = {}; var unusedL10nTokens = {}; @@ -44,19 +46,15 @@ module.exports = function handleUnusedL10n(tasks){ { var tokenNames = unusedL10nTokens[dictFileName]; - dictFileName = dictFileName.slice(process.cwd().length); - - var key = 'unused l10n tokens at ' + dictFileName; - var task = { name: key, result: { warnings: [] } }; - - for (var tokenName in tokenNames) - { - task.result.warnings.push({ - file: tokenName - }); - } - - tasks.push(task); + dictFileName = path.relative(process.cwd(), dictFileName); + tasks.push({ + name: 'unused l10n tokens at ' + dictFileName, + result: { + warnings: Object.keys(tokenNames).map(function(name){ + return { file: 'Path "' + name + '" defined but never used' }; + }) + } + }); } } }; diff --git a/lib/lint/unused/l10n.js b/lib/lint/unused/l10n.js index 0ff4211..5aefd22 100644 --- a/lib/lint/unused/l10n.js +++ b/lib/lint/unused/l10n.js @@ -93,7 +93,7 @@ exports.warn = function(flow){ if (!tokenNames[tokenName]) { flow.warn({ file: dictFileName, - message: tokenName + message: 'Path "' + tokenName + '" defined but never used' }); } }