tools/lib/docs_validator.js - cordova-docs - Git at Google

 /*
        Licensed to the Apache Software Foundation (ASF) under one
        or more contributor license agreements.  See the NOTICE file
        distributed with this work for additional information
        regarding copyright ownership.  The ASF licenses this file
        to you under the Apache License, Version 2.0 (the
        "License"); you may not use this file except in compliance
        with the License.  You may obtain a copy of the License at

          http://www.apache.org/licenses/LICENSE-2.0

        Unless required by applicable law or agreed to in writing,
        software distributed under the License is distributed on an
        "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
        KIND, either express or implied.  See the License for the
        specific language governing permissions and limitations
        under the License.
 */
 /* jslint node: true, continue:true */
 /* global setImmediate */
 var fs = require('fs-extra');
 var path = require('path');
 var FileHelpers = require('./file_helpers');
 var cheerio = require('cheerio');
 var jsdiff = require('diff');
 var yaml = require('js-yaml'); // eslint-disable-line no-unused-vars
 var dir = require('node-dir');
 require('colors');

 var DocsValidator = (function () {
     'use strict';

     function processEachFileSync (source_path, fileCallback, directoryCallback) { // eslint-disable-line no-unused-vars
         var directoryEntries = fs.readdirSync(source_path);
         directoryEntries.forEach(function (dirEntry) {
             var fullPath = path.join(source_path, dirEntry);
             var stat;
             if (!fs.existsSync(fullPath)) {
                 return;
             }

             stat = fs.lstatSync(fullPath);
             if (stat.isFile()) {
                 fileCallback(fullPath);
                 return;
             }

             if (stat.isDirectory()) {
                 if (directoryCallback(fullPath)) {
                     processEachFileSync(fullPath, fileCallback, directoryCallback);
                 }

             }
         });
     }

     function processEachFile (source_path, fileCallback, directoryCallback, errorCallback) { // eslint-disable-line no-unused-vars
         fs.readdirSync(source_path, function (err, directoryEntries) {
             if (err) {
                 errorCallback(err);
                 return;
             }

             directoryEntries.forEach(function (dirEntry) {
                 var fullPath = path.join(source_path, dirEntry);
                 fs.exists(fullPath, function (exists) {
                     if (!exists) {
                         return;
                     }

                     fs.lstat(fullPath, function (err, stat) {
                         if (err) {
                             errorCallback(err);
                             return;
                         }

                         if (stat.isFile()) {
                             fileCallback(fullPath);
                             return;
                         }

                         if (stat.isDirectory()) {
                             if (directoryCallback(fullPath)) {
                                 processEachFile(fullPath, fileCallback, directoryCallback, errorCallback);
                             }

                         }
                     });
                 });
             });
         });
     }

     /**
     * Creates a new instance of DocsValidator
     * @param inputDirectory Directory which contains files which has to be processed.
     */
     function DocsValidator (originalDirectory) {
         this.original_directory = originalDirectory || path.join(FileHelpers.getRootDirectory(), 'docs');
     }

     /**
     * Validates the specific version of documentation
     * @param language Language which has to be validated.
     * @param version Version which files has to be validated.
     * @param verbose_mode Verbosity level.
     */
     DocsValidator.prototype.validate = function (language, version, verbose_mode) {
         var self = this;
         var ignore_list = ['.', '..', '.DS_Store', 'test'];

         verbose_mode = verbose_mode || 0;
         if (verbose_mode > 0) {
             console.log('Comparing docs for lang ' + language + ' and version ' + version);
             console.log('Clearing output directory');
         }

         fs.readdirSync(this.original_directory).forEach(function (language_dir) {
             if (ignore_list.indexOf(language_dir) !== -1) {
                 return;
             }

             if (language && language_dir !== language) {
                 return;
             }

             var language_path = path.join(self.original_directory, language_dir);

             fs.readdirSync(language_path).forEach(function (version_dir) {
                 if (ignore_list.indexOf(version_dir) !== -1) {
                     return;
                 }

                 if (version && version_dir !== version) {
                     return;
                 }

                 var input_path = path.join(self.original_directory, language_dir, version_dir);
                 var options = {
                     lang: language_dir,
                     version: version_dir,
                     verbose: verbose_mode
                 };

                 console.log(' => Validating the Cordova Documentation for ' + version_dir + '-' + language_dir + '...');
                 self.process(input_path, options);
             });
         });
     };

     /**
     * Validates the specific version of documentation
     * @param outputDirectory Directory where documentation is stored.
     * @param language Language which has to be validated.
     * @param version Version which files has to be validated.
     * @param verbose_mode Verbosity level.
     */
     DocsValidator.prototype.validateTranslation = function (docsDirectory, language, version, verbose_mode) {
         var self = this;
         var outputDirectory = path.resolve(docsDirectory || FileHelpers.getDefaultOutputDirectory());
         var ignore_list = ['.', '..', '.DS_Store', 'test'];

         verbose_mode = verbose_mode || 0;
         if (verbose_mode > 0) {
             console.log('Comparing docs for lang ' + language + ' and version ' + version);
             console.log('Clearing output directory');
         }

         fs.readdirSync(outputDirectory).forEach(function (language_dir) {
             if (ignore_list.indexOf(language_dir) !== -1) {
                 return;
             }

             if (language && language_dir !== language) {
                 return;
             }

             var language_path = path.join(outputDirectory, language_dir);

             fs.readdirSync(language_path).forEach(function (version_dir) {
                 if (ignore_list.indexOf(version_dir) !== -1) {
                     return;
                 }

                 if (version && version_dir !== version) {
                     return;
                 }

                 var input_path = path.join(outputDirectory, language_dir, version_dir);
                 var source_path = path.join(outputDirectory, 'en', version_dir);
                 var options = {
                     lang: language_dir,
                     version: version_dir,
                     verbose: verbose_mode
                 };

                 console.log(' => Validating translation for version ' + version_dir + ' on language ' + language_dir + '...');
                 self.doValidateTranslation(source_path, input_path, options);
             });
         });
     };
     DocsValidator.prototype.doValidateTranslation = function (original_directory, comparing_directory, options) {
         var self = this;
         var compareFiles; // eslint-disable-line no-unused-vars
         var completed;
         console.log('Comparing ' + original_directory);
         console.log('with ' + comparing_directory);
         completed = false;
         dir.readFiles(original_directory,
             { match: /\.html/ },
             function (err, content, filename, next) {
                 if (err) {
                     throw err;
                 }

                 var relativePath = path.relative(original_directory, filename);
                 var alternativeFile = path.join(comparing_directory, relativePath);
                 var $ = cheerio.load(alternativeFile); // eslint-disable-line no-unused-vars
                 fs.readFile(alternativeFile, function (err, data) {
                     if (err) {
                         throw err;
                     }

                     var target = cheerio.load(data);
                     var source = cheerio.load(content);

                     self.validateLinksStructure(relativePath, source, target, options);
                     self.validateDomStructure(relativePath, source, target, options);
                     next();
                 });
             },
             function (err, files) {
                 if (err) {
                     throw err;
                 }

                 completed = true;
             });
         function waitCompletition () {
             if (!completed) {
                 setImmediate(waitCompletition);
             }
         }

         setImmediate(waitCompletition);
     };
     DocsValidator.prototype.process = function (original_directory, options) {
         var self = this;
         var compareFiles; // eslint-disable-line no-unused-vars
         var completed;
         console.log('Processing ' + original_directory);
         compareFiles = function (fileName) {
             self.validateYaml(fileName, options);
         };
         completed = false;
         dir.readFiles(original_directory,
             { match: /\.md$/ },
             function (err, content, filename, next) {
                 if (err) {
                     throw err;
                 }

                 self.validateYaml(filename, content, options);
                 next();
             },
             function (err, files) {
                 if (err) {
                     throw err;
                 }

                 completed = true;
             });
         function waitCompletition () {
             if (!completed) {
                 setImmediate(waitCompletition);
             }
         }

         setImmediate(waitCompletition);
     };

     DocsValidator.prototype.validateDomStructure = function (relativePath, source, target, options) {
         var sourceDom = source('#content');
         var targetDom = target('#content');
         var sourceDomList = '';
         var targetDomList = '';
         var sourceDomHtmlList = [];
         var targetDomHtmlList = [];
         var changes;
         var changed = false; // eslint-disable-line no-unused-vars
         function convertSource (element, initial, offset) {
             var i;
             var child;
             if (element.children === undefined) {
                 console.log(element);
             }

             for (i = 0; i < element.children.length; i += 1) {
                 child = element.children[i];
                 if (child.type !== 'tag') {
                     continue;
                 }

                 initial += offset + child.name + '\r\n';
                 initial = convertSource(child, initial, ' ' + offset);
             }

             return initial;
         }
         function convertSourceHtml (element, initial, offset) {
             var i;
             var child;
             if (element.children === undefined) {
                 console.log(element);
             }

             for (i = 0; i < element.children.length; i += 1) {
                 child = element.children[i];
                 if (child.type !== 'tag') {
                     continue;
                 }

                 initial += offset + child.name + '(' + cheerio(child).html() + ')' + '\r\n';
                 // console.log(cheerio(child).html());
                 initial = convertSourceHtml(child, initial, ' ' + offset);
             }

             return initial;
         }
         sourceDomList = convertSource(sourceDom[0], '', '');
         targetDomList = convertSource(targetDom[0], '', '');
         sourceDomHtmlList = convertSourceHtml(sourceDom[0], '', '').split('\r\n') || [];
         targetDomHtmlList = convertSourceHtml(targetDom[0], '', '').split('\r\n') || [];
         if (sourceDomList !== targetDomList) {
             console.error('Path ' + relativePath + ' has different dom structure.');
             if (options.verbose > 0) {
                 // console.log(jsdiff.createPatch(relativePath, sourceDomList, targetDomList, '', ''));
                 changes = jsdiff.diffLines(sourceDomList, targetDomList);
                 if (options.verbose > 0) {
                     var sourceLinesCounter = 0;
                     var targetLinesCounter = 0;
                     changes.forEach(function (part) {
                         // green for additions, red for deletions
                         // grey for common parts
                         var color = part.added ? 'green' : (part.removed ? 'red' : 'grey');
                         var value = part.value;
                         // process.stderr.write(value[color]);
                         if (part.added) {
                             value = targetDomHtmlList.slice(targetLinesCounter, targetLinesCounter + part.count).join('\r\n') + '\r\n';
                             targetLinesCounter += part.count;
                         } else if (part.removed) {
                             value = sourceDomHtmlList.slice(sourceLinesCounter, sourceLinesCounter + part.count).join('\r\n') + '\r\n';
                             sourceLinesCounter += part.count;
                         } else {
                             sourceLinesCounter += part.count;
                             targetLinesCounter += part.count;
                             var contextLength = 3;
                             if (part.count > contextLength * 2 + 1) {
                                 value = part.value.split('\r\n').slice(0, contextLength).concat(['...\r\n'], part.value.split('\r\n').slice(part.count - contextLength, part.count)).join('\r\n') + '\r\n';
                             } else {
                                 value = part.value;
                             }
                         }
                         process.stderr.write(value[color]);
                     });

                     console.log();
                 }
             }
         }
     };

     DocsValidator.prototype.validateLinksStructure = function (relativePath, source, target, options) {
         // Skip _index.html since it will have links in the different
         // order, not as in the original docs, since each word
         // will be translated to different languages.
         if (relativePath === '_index.html') {
             return;
         }

         var sourceLinks = source('#content a');
         var targetLinks = target('#content a');
         var sourceLinksList = '';
         var targetLinksList = '';
         var changes;
         var changed = false;
         sourceLinks.each(function (i, a) {
             var link = a.attribs.href || '';
             link = link.split('#')[0];
             if (link) {
                 sourceLinksList += link + '\n';
             }
         });
         targetLinks.each(function (i, a) {
             var link = a.attribs.href || '';
             link = link.split('#')[0];
             if (link) {
                 targetLinksList += link + '\n';
             }
         });
         changes = jsdiff.diffLines(sourceLinksList, targetLinksList);
         changes.forEach(function (part) {
             changed = part.added || part.removed;
         });
         if (changed) {
             console.error('Path ' + relativePath + ' has different links.');
             if (options.verbose > 0) {
                 changes.forEach(function (part) {
                     // green for additions, red for deletions
                     // grey for common parts
                     var color = part.added ? 'green' : (part.removed ? 'red' : 'grey');
                     process.stderr.write(part.value[color]);
                 });

                 console.log();
             }
         }
     };

     DocsValidator.prototype.validateYaml = function (sourceFile, content, options) {
         if (options.verbose > 0) {
             console.log('Validate ' + sourceFile);
         }

         var yamlRegexStripper = /^(---\s*\n[\s\S]*?\n?)^(---\s*$\n?)/m;
         var match = yamlRegexStripper.exec(content);

         if (!match) {
             console.log('File ' + sourceFile + ' miss the YAML license header');
             return 1;
         } else {
             if (match[1].indexOf('license:') === -1) {
                 console.log('File ' + sourceFile + ' has invalid YAML license header');
                 return 2;
             }
         }

         return 0;
     };

     /**
     * Validates the specific version of documentation
     * @param outputDirectory Directory where documentation is stored.
     * @param language Language which has to be validated.
     * @param version Version which files has to be validated.
     * @param verbose_mode Verbosity level.
     */
     DocsValidator.prototype.fixYamlHeader = function (docsDirectory, language, version, verbose_mode) {
         var self = this;
         var outputDirectory = path.resolve(docsDirectory || FileHelpers.getDefaultInputDirectory());
         var ignore_list = ['.', '..', '.DS_Store', 'test'];

         verbose_mode = verbose_mode || 0;
         if (verbose_mode > 0) {
             console.log('Fixing YAML headers for lang ' + language + ' and version ' + version);
             console.log('Clearing output directory');
         }

         fs.readdirSync(outputDirectory).forEach(function (language_dir) {
             if (ignore_list.indexOf(language_dir) !== -1) {
                 return;
             }

             if (language && language_dir !== language) {
                 return;
             }

             var language_path = path.join(outputDirectory, language_dir);

             fs.readdirSync(language_path).forEach(function (version_dir) {
                 if (ignore_list.indexOf(version_dir) !== -1) {
                     return;
                 }

                 if (version && version_dir !== version) {
                     return;
                 }

                 var input_path = path.join(outputDirectory, language_dir, version_dir);
                 var options = {
                     lang: language_dir,
                     version: version_dir,
                     verbose: verbose_mode
                 };

                 console.log(' => Fix YAML header for version ' + version_dir + ' on language ' + language_dir + '...');
                 self.doFixYamlHeader(input_path, options);
             });
         });
     };
     DocsValidator.prototype.doFixYamlHeader = function (lang_directory, options) {
         var self = this;
         var compareFiles; // eslint-disable-line no-unused-vars
         var completed;
         console.log('Fixing ' + lang_directory);
         completed = false;
         dir.readFiles(lang_directory,
             { match: /\.md/ },
             function (err, content, filename, next) {
                 if (err) {
                     throw err;
                 }

                 var relativePath = path.relative(lang_directory, filename); // eslint-disable-line no-unused-vars
                 fs.readFile(filename, 'utf8', function (err, data) {
                     if (err) {
                         throw err;
                     }

                     var target = data; // eslint-disable-line no-unused-vars
                     var validationResult = self.validateYaml(filename, content, options);
                     var yamlReplaceRegex1;
                     var yamlReplaceRegex2;
                     var eol = require('os').type() === 'win32' ? '\r\n' : '\n';
                     var prefix = '         ';
                     var correctLicense = '---' + eol +
                         'license: Licensed to the Apache Software Foundation (ASF) under one' + eol +
                         prefix + 'or more contributor license agreements.  See the NOTICE file' + eol +
                         prefix + 'distributed with this work for additional information' + eol +
                         prefix + 'regarding copyright ownership.  The ASF licenses this file' + eol +
                         prefix + 'to you under the Apache License, Version 2.0 (the' + eol +
                         prefix + '"License"); you may not use this file except in compliance' + eol +
                         prefix + 'with the License.  You may obtain a copy of the License at' + eol +
                         eol +
                         prefix + '  http://www.apache.org/licenses/LICENSE-2.0' + eol +
                         eol +
                         prefix + 'Unless required by applicable law or agreed to in writing,' + eol +
                         prefix + 'software distributed under the License is distributed on an' + eol +
                         prefix + '"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY' + eol +
                         prefix + 'KIND, either express or implied.  See the License for the' + eol +
                         prefix + 'specific language governing permissions and limitations' + eol +
                         prefix + 'under the License.' + eol +
                         '---' + eol + eol;

                     if (validationResult !== 0) {
                         yamlReplaceRegex1 = /^(\* \* \*\s*\n[\s\S]*?\n?)^(\#\# (under the License\.|unter der Lizenz\.|по лицензии\.|aux termes de la licence\.|con la licenza\.|ライセンス。|라이센스\.|根據許可證。)\s*$\n?)/m; // eslint-disable-line no-useless-escape
                         if (yamlReplaceRegex1.exec(content)) {
                             content = correctLicense + content.replace(yamlReplaceRegex1, '');
                         } else {
                             yamlReplaceRegex2 = /^(\* \* \*\s*\n[\s\S]*?\n?)^(\* \* \*\s*\s*$\n?)/m;
                             if (yamlReplaceRegex2.exec(content)) {
                                 content = correctLicense + content.replace(yamlReplaceRegex2, '');
                             }
                         }

                         fs.writeFile(filename, content, 'utf8', function (err, data) {
                             if (err) {
                                 throw err;
                             }

                             next();
                         });
                     } else {
                         next();
                     }
                 });
             },
             function (err, files) {
                 if (err) {
                     throw err;
                 }

                 completed = true;
             });
         function waitCompletition () {
             if (!completed) {
                 setImmediate(waitCompletition);
             }
         }

         setImmediate(waitCompletition);
     };

     return DocsValidator;
 }());
 module.exports = DocsValidator;
	/*
	Licensed to the Apache Software Foundation (ASF) under one
	or more contributor license agreements. See the NOTICE file
	distributed with this work for additional information
	regarding copyright ownership. The ASF licenses this file
	to you under the Apache License, Version 2.0 (the
	"License"); you may not use this file except in compliance
	with the License. You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

	Unless required by applicable law or agreed to in writing,
	software distributed under the License is distributed on an
	"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	KIND, either express or implied. See the License for the
	specific language governing permissions and limitations
	under the License.
	*/
	/* jslint node: true, continue:true */
	/* global setImmediate */
	var fs = require('fs-extra');
	var path = require('path');
	var FileHelpers = require('./file_helpers');
	var cheerio = require('cheerio');
	var jsdiff = require('diff');
	var yaml = require('js-yaml'); // eslint-disable-line no-unused-vars
	var dir = require('node-dir');
	require('colors');

	var DocsValidator = (function () {
	'use strict';

	function processEachFileSync (source_path, fileCallback, directoryCallback) { // eslint-disable-line no-unused-vars
	var directoryEntries = fs.readdirSync(source_path);
	directoryEntries.forEach(function (dirEntry) {
	var fullPath = path.join(source_path, dirEntry);
	var stat;
	if (!fs.existsSync(fullPath)) {
	return;
	}

	stat = fs.lstatSync(fullPath);
	if (stat.isFile()) {
	fileCallback(fullPath);
	return;
	}

	if (stat.isDirectory()) {
	if (directoryCallback(fullPath)) {
	processEachFileSync(fullPath, fileCallback, directoryCallback);
	}

	}
	});
	}

	function processEachFile (source_path, fileCallback, directoryCallback, errorCallback) { // eslint-disable-line no-unused-vars
	fs.readdirSync(source_path, function (err, directoryEntries) {
	if (err) {
	errorCallback(err);
	return;
	}

	directoryEntries.forEach(function (dirEntry) {
	var fullPath = path.join(source_path, dirEntry);
	fs.exists(fullPath, function (exists) {
	if (!exists) {
	return;
	}

	fs.lstat(fullPath, function (err, stat) {
	if (err) {
	errorCallback(err);
	return;
	}

	if (stat.isFile()) {
	fileCallback(fullPath);
	return;
	}

	if (stat.isDirectory()) {
	if (directoryCallback(fullPath)) {
	processEachFile(fullPath, fileCallback, directoryCallback, errorCallback);
	}

	}
	});
	});
	});
	});
	}

	/**
	* Creates a new instance of DocsValidator
	* @param inputDirectory Directory which contains files which has to be processed.
	*/
	function DocsValidator (originalDirectory) {
	this.original_directory = originalDirectory \|\| path.join(FileHelpers.getRootDirectory(), 'docs');
	}

	/**
	* Validates the specific version of documentation
	* @param language Language which has to be validated.
	* @param version Version which files has to be validated.
	* @param verbose_mode Verbosity level.
	*/
	DocsValidator.prototype.validate = function (language, version, verbose_mode) {
	var self = this;
	var ignore_list = ['.', '..', '.DS_Store', 'test'];

	verbose_mode = verbose_mode \|\| 0;
	if (verbose_mode > 0) {
	console.log('Comparing docs for lang ' + language + ' and version ' + version);
	console.log('Clearing output directory');
	}

	fs.readdirSync(this.original_directory).forEach(function (language_dir) {
	if (ignore_list.indexOf(language_dir) !== -1) {
	return;
	}

	if (language && language_dir !== language) {
	return;
	}

	var language_path = path.join(self.original_directory, language_dir);

	fs.readdirSync(language_path).forEach(function (version_dir) {
	if (ignore_list.indexOf(version_dir) !== -1) {
	return;
	}

	if (version && version_dir !== version) {
	return;
	}

	var input_path = path.join(self.original_directory, language_dir, version_dir);
	var options = {
	lang: language_dir,
	version: version_dir,
	verbose: verbose_mode
	};

	console.log(' => Validating the Cordova Documentation for ' + version_dir + '-' + language_dir + '...');
	self.process(input_path, options);
	});
	});
	};

	/**
	* Validates the specific version of documentation
	* @param outputDirectory Directory where documentation is stored.
	* @param language Language which has to be validated.
	* @param version Version which files has to be validated.
	* @param verbose_mode Verbosity level.
	*/
	DocsValidator.prototype.validateTranslation = function (docsDirectory, language, version, verbose_mode) {
	var self = this;
	var outputDirectory = path.resolve(docsDirectory \|\| FileHelpers.getDefaultOutputDirectory());
	var ignore_list = ['.', '..', '.DS_Store', 'test'];

	verbose_mode = verbose_mode \|\| 0;
	if (verbose_mode > 0) {
	console.log('Comparing docs for lang ' + language + ' and version ' + version);
	console.log('Clearing output directory');
	}

	fs.readdirSync(outputDirectory).forEach(function (language_dir) {
	if (ignore_list.indexOf(language_dir) !== -1) {
	return;
	}

	if (language && language_dir !== language) {
	return;
	}

	var language_path = path.join(outputDirectory, language_dir);

	fs.readdirSync(language_path).forEach(function (version_dir) {
	if (ignore_list.indexOf(version_dir) !== -1) {
	return;
	}

	if (version && version_dir !== version) {
	return;
	}

	var input_path = path.join(outputDirectory, language_dir, version_dir);
	var source_path = path.join(outputDirectory, 'en', version_dir);
	var options = {
	lang: language_dir,
	version: version_dir,
	verbose: verbose_mode
	};

	console.log(' => Validating translation for version ' + version_dir + ' on language ' + language_dir + '...');
	self.doValidateTranslation(source_path, input_path, options);
	});
	});
	};
	DocsValidator.prototype.doValidateTranslation = function (original_directory, comparing_directory, options) {
	var self = this;
	var compareFiles; // eslint-disable-line no-unused-vars
	var completed;
	console.log('Comparing ' + original_directory);
	console.log('with ' + comparing_directory);
	completed = false;
	dir.readFiles(original_directory,
	{ match: /\.html/ },
	function (err, content, filename, next) {
	if (err) {
	throw err;
	}

	var relativePath = path.relative(original_directory, filename);
	var alternativeFile = path.join(comparing_directory, relativePath);
	var $ = cheerio.load(alternativeFile); // eslint-disable-line no-unused-vars
	fs.readFile(alternativeFile, function (err, data) {
	if (err) {
	throw err;
	}

	var target = cheerio.load(data);
	var source = cheerio.load(content);

	self.validateLinksStructure(relativePath, source, target, options);
	self.validateDomStructure(relativePath, source, target, options);
	next();
	});
	},
	function (err, files) {
	if (err) {
	throw err;
	}

	completed = true;
	});
	function waitCompletition () {
	if (!completed) {
	setImmediate(waitCompletition);
	}
	}

	setImmediate(waitCompletition);
	};
	DocsValidator.prototype.process = function (original_directory, options) {
	var self = this;
	var compareFiles; // eslint-disable-line no-unused-vars
	var completed;
	console.log('Processing ' + original_directory);
	compareFiles = function (fileName) {
	self.validateYaml(fileName, options);
	};
	completed = false;
	dir.readFiles(original_directory,
	{ match: /\.md$/ },
	function (err, content, filename, next) {
	if (err) {
	throw err;
	}

	self.validateYaml(filename, content, options);
	next();
	},
	function (err, files) {
	if (err) {
	throw err;
	}

	completed = true;
	});
	function waitCompletition () {
	if (!completed) {
	setImmediate(waitCompletition);
	}
	}

	setImmediate(waitCompletition);
	};

	DocsValidator.prototype.validateDomStructure = function (relativePath, source, target, options) {
	var sourceDom = source('#content');
	var targetDom = target('#content');
	var sourceDomList = '';
	var targetDomList = '';
	var sourceDomHtmlList = [];
	var targetDomHtmlList = [];
	var changes;
	var changed = false; // eslint-disable-line no-unused-vars
	function convertSource (element, initial, offset) {
	var i;
	var child;
	if (element.children === undefined) {
	console.log(element);
	}

	for (i = 0; i < element.children.length; i += 1) {
	child = element.children[i];
	if (child.type !== 'tag') {
	continue;
	}

	initial += offset + child.name + '\r\n';
	initial = convertSource(child, initial, ' ' + offset);
	}

	return initial;
	}
	function convertSourceHtml (element, initial, offset) {
	var i;
	var child;
	if (element.children === undefined) {
	console.log(element);
	}

	for (i = 0; i < element.children.length; i += 1) {
	child = element.children[i];
	if (child.type !== 'tag') {
	continue;
	}

	initial += offset + child.name + '(' + cheerio(child).html() + ')' + '\r\n';
	// console.log(cheerio(child).html());
	initial = convertSourceHtml(child, initial, ' ' + offset);
	}

	return initial;
	}
	sourceDomList = convertSource(sourceDom[0], '', '');
	targetDomList = convertSource(targetDom[0], '', '');
	sourceDomHtmlList = convertSourceHtml(sourceDom[0], '', '').split('\r\n') \|\| [];
	targetDomHtmlList = convertSourceHtml(targetDom[0], '', '').split('\r\n') \|\| [];
	if (sourceDomList !== targetDomList) {
	console.error('Path ' + relativePath + ' has different dom structure.');
	if (options.verbose > 0) {
	// console.log(jsdiff.createPatch(relativePath, sourceDomList, targetDomList, '', ''));
	changes = jsdiff.diffLines(sourceDomList, targetDomList);
	if (options.verbose > 0) {
	var sourceLinesCounter = 0;
	var targetLinesCounter = 0;
	changes.forEach(function (part) {
	// green for additions, red for deletions
	// grey for common parts
	var color = part.added ? 'green' : (part.removed ? 'red' : 'grey');
	var value = part.value;
	// process.stderr.write(value[color]);
	if (part.added) {
	value = targetDomHtmlList.slice(targetLinesCounter, targetLinesCounter + part.count).join('\r\n') + '\r\n';
	targetLinesCounter += part.count;
	} else if (part.removed) {
	value = sourceDomHtmlList.slice(sourceLinesCounter, sourceLinesCounter + part.count).join('\r\n') + '\r\n';
	sourceLinesCounter += part.count;
	} else {
	sourceLinesCounter += part.count;
	targetLinesCounter += part.count;
	var contextLength = 3;
	if (part.count > contextLength * 2 + 1) {
	value = part.value.split('\r\n').slice(0, contextLength).concat(['...\r\n'], part.value.split('\r\n').slice(part.count - contextLength, part.count)).join('\r\n') + '\r\n';
	} else {
	value = part.value;
	}
	}
	process.stderr.write(value[color]);
	});

	console.log();
	}
	}
	}
	};

	DocsValidator.prototype.validateLinksStructure = function (relativePath, source, target, options) {
	// Skip _index.html since it will have links in the different
	// order, not as in the original docs, since each word
	// will be translated to different languages.
	if (relativePath === '_index.html') {
	return;
	}

	var sourceLinks = source('#content a');
	var targetLinks = target('#content a');
	var sourceLinksList = '';
	var targetLinksList = '';
	var changes;
	var changed = false;
	sourceLinks.each(function (i, a) {
	var link = a.attribs.href \|\| '';
	link = link.split('#')[0];
	if (link) {
	sourceLinksList += link + '\n';
	}
	});
	targetLinks.each(function (i, a) {
	var link = a.attribs.href \|\| '';
	link = link.split('#')[0];
	if (link) {
	targetLinksList += link + '\n';
	}
	});
	changes = jsdiff.diffLines(sourceLinksList, targetLinksList);
	changes.forEach(function (part) {
	changed = part.added \|\| part.removed;
	});
	if (changed) {
	console.error('Path ' + relativePath + ' has different links.');
	if (options.verbose > 0) {
	changes.forEach(function (part) {
	// green for additions, red for deletions
	// grey for common parts
	var color = part.added ? 'green' : (part.removed ? 'red' : 'grey');
	process.stderr.write(part.value[color]);
	});

	console.log();
	}
	}
	};

	DocsValidator.prototype.validateYaml = function (sourceFile, content, options) {
	if (options.verbose > 0) {
	console.log('Validate ' + sourceFile);
	}

	var yamlRegexStripper = /^(---\s\n[\s\S]?\n?)^(---\s*$\n?)/m;
	var match = yamlRegexStripper.exec(content);

	if (!match) {
	console.log('File ' + sourceFile + ' miss the YAML license header');
	return 1;
	} else {
	if (match[1].indexOf('license:') === -1) {
	console.log('File ' + sourceFile + ' has invalid YAML license header');
	return 2;
	}
	}

	return 0;
	};

	/**
	* Validates the specific version of documentation
	* @param outputDirectory Directory where documentation is stored.
	* @param language Language which has to be validated.
	* @param version Version which files has to be validated.
	* @param verbose_mode Verbosity level.
	*/
	DocsValidator.prototype.fixYamlHeader = function (docsDirectory, language, version, verbose_mode) {
	var self = this;
	var outputDirectory = path.resolve(docsDirectory \|\| FileHelpers.getDefaultInputDirectory());
	var ignore_list = ['.', '..', '.DS_Store', 'test'];

	verbose_mode = verbose_mode \|\| 0;
	if (verbose_mode > 0) {
	console.log('Fixing YAML headers for lang ' + language + ' and version ' + version);
	console.log('Clearing output directory');
	}

	fs.readdirSync(outputDirectory).forEach(function (language_dir) {
	if (ignore_list.indexOf(language_dir) !== -1) {
	return;
	}

	if (language && language_dir !== language) {
	return;
	}

	var language_path = path.join(outputDirectory, language_dir);

	fs.readdirSync(language_path).forEach(function (version_dir) {
	if (ignore_list.indexOf(version_dir) !== -1) {
	return;
	}

	if (version && version_dir !== version) {
	return;
	}

	var input_path = path.join(outputDirectory, language_dir, version_dir);
	var options = {
	lang: language_dir,
	version: version_dir,
	verbose: verbose_mode
	};

	console.log(' => Fix YAML header for version ' + version_dir + ' on language ' + language_dir + '...');
	self.doFixYamlHeader(input_path, options);
	});
	});
	};
	DocsValidator.prototype.doFixYamlHeader = function (lang_directory, options) {
	var self = this;
	var compareFiles; // eslint-disable-line no-unused-vars
	var completed;
	console.log('Fixing ' + lang_directory);
	completed = false;
	dir.readFiles(lang_directory,
	{ match: /\.md/ },
	function (err, content, filename, next) {
	if (err) {
	throw err;
	}

	var relativePath = path.relative(lang_directory, filename); // eslint-disable-line no-unused-vars
	fs.readFile(filename, 'utf8', function (err, data) {
	if (err) {
	throw err;
	}

	var target = data; // eslint-disable-line no-unused-vars
	var validationResult = self.validateYaml(filename, content, options);
	var yamlReplaceRegex1;
	var yamlReplaceRegex2;
	var eol = require('os').type() === 'win32' ? '\r\n' : '\n';
	var prefix = ' ';
	var correctLicense = '---' + eol +
	'license: Licensed to the Apache Software Foundation (ASF) under one' + eol +
	prefix + 'or more contributor license agreements. See the NOTICE file' + eol +
	prefix + 'distributed with this work for additional information' + eol +
	prefix + 'regarding copyright ownership. The ASF licenses this file' + eol +
	prefix + 'to you under the Apache License, Version 2.0 (the' + eol +
	prefix + '"License"); you may not use this file except in compliance' + eol +
	prefix + 'with the License. You may obtain a copy of the License at' + eol +
	eol +
	prefix + ' http://www.apache.org/licenses/LICENSE-2.0' + eol +
	eol +
	prefix + 'Unless required by applicable law or agreed to in writing,' + eol +
	prefix + 'software distributed under the License is distributed on an' + eol +
	prefix + '"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY' + eol +
	prefix + 'KIND, either express or implied. See the License for the' + eol +
	prefix + 'specific language governing permissions and limitations' + eol +
	prefix + 'under the License.' + eol +
	'---' + eol + eol;

	if (validationResult !== 0) {
	yamlReplaceRegex1 = /^(\* \* \\s\n[\s\S]?\n?)^(\#\# (under the License\.\|unter der Lizenz\.\|по лицензии\.\|aux termes de la licence\.\|con la licenza\.\|ライセンス。\|라이센스\.\|根據許可證。)\s$\n?)/m; // eslint-disable-line no-useless-escape
	if (yamlReplaceRegex1.exec(content)) {
	content = correctLicense + content.replace(yamlReplaceRegex1, '');
	} else {
	yamlReplaceRegex2 = /^(\* \* \\s\n[\s\S]?\n?)^(\ \* \\s\s*$\n?)/m;
	if (yamlReplaceRegex2.exec(content)) {
	content = correctLicense + content.replace(yamlReplaceRegex2, '');
	}
	}

	fs.writeFile(filename, content, 'utf8', function (err, data) {
	if (err) {
	throw err;
	}

	next();
	});
	} else {
	next();
	}
	});
	},
	function (err, files) {
	if (err) {
	throw err;
	}

	completed = true;
	});
	function waitCompletition () {
	if (!completed) {
	setImmediate(waitCompletition);
	}
	}

	setImmediate(waitCompletition);
	};

	return DocsValidator;
	}());
	module.exports = DocsValidator;