| const fs = require('fs'); |
| const globby = require('globby'); |
| const path = require('path'); |
| const puppeteer = require('puppeteer'); |
| const matter = require('gray-matter'); |
| const argparse = require('argparse'); |
| const minimatch = require('minimatch'); |
| const {execFile} = require('child_process'); |
| const cwebpBin = require('cwebp-bin'); |
| const util = require('util'); |
| const chalk = require('chalk'); |
| const sharp = require('sharp'); |
| const fse = require('fs-extra'); |
| const { compareImage } = require('../common/compareImage'); |
| const { runTasks } = require('../common/task'); |
| const nStatic = require('node-static'); |
| const shell = require('shelljs'); |
| // const { createFFmpeg, fetchFile } = require('@ffmpeg/ffmpeg'); |
| |
| function optionToJson(obj, prop) { |
| let json = JSON.stringify(obj, function(key, value) { |
| if (typeof value === 'function') { |
| return 'expr: ' + value.toString(); |
| } |
| return value; |
| }, 2); |
| return json; |
| }; |
| function codeSize(code) { |
| return Buffer.byteLength(code, 'utf-8'); |
| } |
| |
| const parser = new argparse.ArgumentParser({ |
| addHelp: true |
| }); |
| parser.addArgument(['-s', '--source'], { |
| help: 'Source folder' |
| }); |
| parser.addArgument(['-t', '--theme'], { |
| help: 'Theme list, default to be all' |
| }); |
| parser.addArgument(['-p', '--pattern'], { |
| help: 'Glob match patterns for generating thumb. https://github.com/isaacs/minimatch Mutiple match pattens can be splitted with ,' |
| }); |
| parser.addArgument(['--no-thumb'], { |
| help: 'If not generate thumbs', |
| action: 'storeTrue' |
| }); |
| |
| const args = parser.parseArgs(); |
| const sourceFolder = args.source || 'data'; |
| let themeList = args.theme || 'default,dark'; |
| let matchPattern = args.pattern; |
| if (matchPattern) { |
| matchPattern = matchPattern.split(','); |
| } |
| themeList = themeList.split(','); |
| |
| function waitTime(time) { |
| return new Promise((resolve) => setTimeout(resolve, time)); |
| } |
| |
| const BUILD_THUMBS = !args.no_thumb; |
| const DEFAULT_PAGE_WIDTH = 700; |
| const DEFAULT_PAGE_RATIO = 0.75; |
| const OUTPUT_IMAGE_WIDTH = 600; |
| const OUTPUT_IMAGE_HEIGHT = OUTPUT_IMAGE_WIDTH * DEFAULT_PAGE_RATIO; |
| |
| const PORT = 3323; |
| const BASE_URL = `http://localhost:${PORT}`; |
| const SCREENSHOT_PAGE_URL = `${BASE_URL}/tool/screenshot.html`; |
| |
| const IGNORE_LOG = [ |
| // For BMap |
| 'A cookie associated with a cross-site resource at', |
| 'A parser-blocking, cross site', |
| // For ECharts GL |
| 'RENDER WARNING', |
| 'GL ERROR', |
| 'GL_INVALID_OPERATION' |
| ]; |
| |
| function checkHasVideo(videoStart, videoEnd) { |
| return !isNaN(videoStart) && !isNaN(videoEnd) && +videoEnd > +videoStart; |
| } |
| |
| async function convertToWebP(filePath) { |
| return util.promisify(execFile)(cwebpBin, [filePath, '-o', filePath.replace(/\.png$/, '.webp')]); |
| } |
| |
| async function takeScreenshot( |
| browser, |
| ffmpeg, |
| theme, |
| rootDir, |
| basename, |
| hasVideo, |
| // Shot parameters |
| { shotWidth, shotDelay, videoStart, videoEnd } |
| ) { |
| const thumbFolder = (theme !== 'default') ? ('thumb-' + theme) : 'thumb'; |
| const page = await browser.newPage(); |
| const thumbDir = `${rootDir}public/${sourceFolder}/${thumbFolder}`; |
| const fileBase = `${thumbDir}/${basename}`; |
| const webmFile = `${fileBase}.webm`; |
| |
| function checkDownloadFile() { |
| return new Promise(resolve => { |
| let timeout = 0 |
| function check() { |
| if (fs.existsSync(webmFile)) { |
| resolve(); |
| return; |
| } |
| timeout += 100; |
| if (timeout >= 20000 + +videoEnd) { |
| console.error(fileBase + '.webm download timeout.'); |
| resolve(); |
| return; |
| } |
| |
| setTimeout(check, 100); |
| } |
| setTimeout(check, 100); |
| }); |
| } |
| |
| let checkingDownload; |
| |
| await page.setViewport({ |
| width: (shotWidth || DEFAULT_PAGE_WIDTH), |
| height: (shotWidth || DEFAULT_PAGE_WIDTH) * DEFAULT_PAGE_RATIO |
| }); |
| let url = `${SCREENSHOT_PAGE_URL}?c=${basename}&t=${theme}&s=${sourceFolder}`; |
| |
| if (hasVideo) { |
| url += `&start=${videoStart}&end=${videoEnd}`; |
| await page._client.send('Page.setDownloadBehavior', { |
| behavior: 'allow', |
| downloadPath: thumbDir |
| }); |
| |
| checkingDownload = checkDownloadFile(); |
| } |
| |
| const resourceRootPath = `${BASE_URL}/public`; |
| // console.log(url); |
| await page.evaluateOnNewDocument(function (resourceRootPath) { |
| window.ROOT_PATH = resourceRootPath; |
| }, resourceRootPath); |
| |
| page.on('pageerror', function (err) { |
| console.error(chalk.red('[pageerror in]', url)); |
| console.error(chalk.red(err.toString())); |
| }); |
| page.on('console', msg => { |
| const text = msg.text(); |
| if (!IGNORE_LOG.find(a => text.indexOf(a) >= 0)) { |
| console.log(chalk.gray(`PAGE LOG[${basename}]: ${text}`)); |
| } |
| }); |
| |
| console.log(`Generating ${theme} thumbs.....${basename}`); |
| // https://stackoverflow.com/questions/46160929/puppeteer-wait-for-all-images-to-load-then-take-screenshot |
| try { |
| try { |
| await page.goto(url, { |
| waitUntil: 'networkidle0', |
| timeout: 20000 |
| }); |
| } |
| catch (e) { |
| console.error(chalk.red(e)); |
| // Timeout |
| } |
| await waitTime(200); |
| await waitTime(shotDelay || 0); |
| const filePathTmpRaw = `${fileBase}-tmp-raw.png`; |
| const filePathTmp = `${fileBase}-tmp.png`; |
| const filePath = `${fileBase}.png`; |
| |
| fse.ensureDirSync(thumbDir); |
| |
| // Save option for further tests. |
| try { |
| const option = await page.evaluate(() => { |
| return _$getEChartsOption() |
| }); |
| const optionStr = optionToJson(option); |
| fse.ensureDirSync(`${rootDir}public/${sourceFolder}/option/`); |
| fs.writeFileSync(`${rootDir}public/${sourceFolder}/option/${basename}.json`, optionStr, 'utf-8'); |
| } |
| catch (e) { |
| console.error(chalk.red('Failed to generate option')); |
| console.error(chalk.red(e)); |
| } |
| |
| await page.screenshot({ |
| path: filePathTmpRaw, |
| type: 'png' |
| }); |
| |
| await sharp(filePathTmpRaw) |
| .resize(OUTPUT_IMAGE_WIDTH, OUTPUT_IMAGE_HEIGHT) |
| .toFile(filePathTmp); |
| |
| const {diffRatio} = await compareImage(filePath, filePathTmp, 0.1); |
| |
| console.log(filePath); |
| if (diffRatio < 0.01) { |
| console.log('Not changed'); |
| } |
| else { |
| console.log(diffRatio); |
| fs.copyFileSync(filePathTmp, filePath); |
| if (!hasVideo) { |
| await convertToWebP(filePath); |
| } |
| } |
| |
| try { |
| fs.unlinkSync(filePathOld); |
| } |
| catch (e) {} |
| |
| fs.unlinkSync(filePathTmpRaw); |
| fs.unlinkSync(filePathTmp); |
| |
| if (hasVideo) { |
| await checkingDownload; |
| // const webpFile = `${fileBase}.webp`; |
| // const fileContent = fs.readFileSync(webmFile); |
| // ffmpeg.FS('writeFile', `${basename}.webm`, await fetchFile(fileContent)); |
| // await ffmpeg.run('-i', `${basename}.webm`, '-f', 'webp', '-s', `${OUTPUT_IMAGE_WIDTH}x${OUTPUT_IMAGE_HEIGHT}`, `${basename}.webp`); |
| // fs.writeFileSync(webpFile, ffmpeg.FS('readFile', `${basename}.webp`)); |
| // ffmpeg.FS("unlink", `${basename}.webm`) |
| // ffmpeg.FS("unlink", `${basename}.webp`) |
| shell.exec(`ffmpeg -y -i ${fileBase}.webm -s ${OUTPUT_IMAGE_WIDTH}x${OUTPUT_IMAGE_HEIGHT} -f webp ${fileBase}.webp`); |
| try { |
| fs.unlinkSync(webmFile); |
| } |
| catch(e) {} |
| } |
| } |
| catch (e) { |
| console.error(url); |
| console.error(e.toString()); |
| } |
| await page.close(); |
| } |
| |
| (async () => { |
| |
| const rootDir = path.join(__dirname, '../'); |
| // TODO puppeteer will have Navigation Timeout Exceeded: 30000ms exceeded error in these examples. |
| const screenshotBlackList = []; |
| |
| const files = await globby(`${rootDir}public/${sourceFolder}/*.js`); |
| |
| const exampleList = []; |
| |
| let thumbTasks = []; |
| |
| for (let theme of themeList) { |
| for (let fileName of files) { |
| const basename = path.basename(fileName, '.js'); |
| |
| // Remove mapbox temporary |
| if (basename.indexOf('mapbox') >= 0 |
| || basename.indexOf('shanghai') >= 0 |
| || basename === 'lines3d-taxi-routes-of-cape-town' |
| || basename === 'lines3d-taxi-chengdu' |
| || basename === 'map3d-colorful-cities' |
| // TODO Examples that can't work temporary. |
| || basename === 'bar3d-music-visualization' |
| ) { |
| return; |
| } |
| |
| let fmResult; |
| try { |
| const code = fs.readFileSync(`${rootDir}public/${sourceFolder}/${basename}.js`, 'utf-8'); |
| fmResult = matter(code, { |
| delimiters: ['/*', '*/'] |
| }); |
| } |
| catch (e) { |
| fmResult = { |
| data: {} |
| }; |
| } |
| |
| try { |
| const difficulty = fmResult.data.difficulty != null ? fmResult.data.difficulty : 10; |
| const category = (fmResult.data.category || '').split(/,/g).map(a => a.trim()).filter(a => !!a); |
| if (!exampleList.find(item => item.id === basename)) { // Avoid add mulitple times when has multiple themes. |
| exampleList.push({ |
| category: category, |
| id: basename, |
| tags: (fmResult.data.tags || '').split(/,/g).map(a => a.trim()).filter(a => !!a), |
| theme: fmResult.data.theme, |
| title: fmResult.data.title, |
| titleCN: fmResult.data.titleCN, |
| difficulty: +difficulty |
| }); |
| } |
| } |
| catch (e) { |
| throw new Error(e.toString()); |
| } |
| |
| if ( |
| !matchPattern || matchPattern.some(function (pattern) { |
| return minimatch(basename, pattern); |
| }) && screenshotBlackList.indexOf(basename) < 0 |
| ) { |
| thumbTasks.push({ |
| theme, |
| fmResult, |
| basename |
| }); |
| } |
| } |
| } |
| |
| exampleList.sort(function (a, b) { |
| if (a.difficulty === b.difficulty) { |
| return a.id.localeCompare(b.id); |
| } |
| return a.difficulty - b.difficulty; |
| }); |
| |
| const code = ` |
| /* eslint-disable */ |
| // THIS FILE IS GENERATED, DON'T MODIFY */ |
| export default ${JSON.stringify(exampleList, null, 2)}`; |
| |
| if (!matchPattern) { |
| fs.writeFileSync(path.join(__dirname, `../src/data/chart-list-${sourceFolder}.js`), code, 'utf-8'); |
| } |
| |
| // Do screenshot |
| if (BUILD_THUMBS) { |
| const fileServer = new nStatic.Server(rootDir); |
| const server = BUILD_THUMBS && require('http').createServer(function (request, response) { |
| request.addListener('end', function () { |
| fileServer.serve(request, response); |
| }).resume(); |
| }) |
| server && server.listen(PORT); |
| |
| const browser = await puppeteer.launch({ |
| headless: false, |
| args: [ |
| '--headless', |
| '--hide-scrollbars', |
| // https://github.com/puppeteer/puppeteer/issues/4913 |
| '--use-gl=egl', |
| '--mute-audio' |
| ] |
| }); |
| |
| let ffmpeg; |
| // const ffmpeg = createFFmpeg({ log: true }); |
| // await ffmpeg.load(); |
| |
| try { |
| // Take screenshots |
| const animationTasks = thumbTasks.filter(task => { |
| return checkHasVideo(task.fmResult.data.videoStart, task.fmResult.data.videoEnd) |
| }) |
| const staticTasks = thumbTasks.filter(task => { |
| return !checkHasVideo(task.fmResult.data.videoStart, task.fmResult.data.videoEnd) |
| }) |
| await runTasks(staticTasks, async ({basename, fmResult, theme}) => { |
| await takeScreenshot( |
| browser, |
| ffmpeg, |
| theme, |
| rootDir, |
| basename, |
| false, |
| { |
| shotWidth: fmResult.data.shotWidth, |
| shotDelay: fmResult.data.shotDelay |
| } |
| ); |
| }, sourceFolder === 'data-gl' ? 2 : 16); |
| |
| await runTasks(animationTasks, async ({basename, fmResult, theme}) => { |
| await takeScreenshot( |
| browser, |
| ffmpeg, |
| theme, |
| rootDir, |
| basename, |
| true, |
| { |
| shotWidth: fmResult.data.shotWidth, |
| shotDelay: fmResult.data.shotDelay, |
| videoStart: fmResult.data.videoStart, |
| videoEnd: fmResult.data.videoEnd, |
| } |
| ); |
| }, 1); // Webm download seems has issue used with multithreads |
| } |
| catch (e) { |
| server.close(); |
| await browser.close(); |
| throw new Error(e.toString()); |
| } |
| |
| server.close(); |
| await browser.close(); |
| // ffmpeg.exit(0); |
| } |
| })(); |
| |
| |
| process.on('SIGINT', function() { |
| console.log('Closing'); |
| // Close through ctrl + c; |
| process.exit(); |
| }); |