import path from 'node:path'; import {fs, net, tempDir, zip} from '@appium/support'; import {waitForCondition} from 'asyncbox'; import {spawn} from 'node:child_process'; import { log, walk, DEFAULT_LANGUAGE, performApiRequest, RESOURCES_ROOT, DOCUMENTS_EXT, ORIGINAL_MKDOCS_CONFIG, CROWIN_MKDOCS_CONFIG, } from './crowdin-common.mjs'; import _ from 'lodash'; const BUILD_TIMEOUT_MS = 1000 * 60 * 10; const BUILD_STATUS = { finished: 'finished', created: 'created', inProgress: 'inProgress', canceled: 'canceled', failed: 'failed', }; // Add new languages here whenever needed const CROWDIN_TO_FS_LANGUAGES_MAP = { ja: 'ja', 'zh-CN': 'zh', // de,es-ES,fr,it,ja,pt-BR,uk }; /** * * @returns {Promise} */ async function buildTranslations() { log.info('Building project translations'); const {data: buildData} = await performApiRequest('/translations/builds', { method: 'POST', }); return buildData.id; } /** * * @param {number} buildId * @param {string} dstPath * @returns {Promise} */ async function downloadTranslations(buildId, dstPath) { log.info(`Waiting up to ${BUILD_TIMEOUT_MS / 1000}s for the build #${buildId} to finish`); await waitForCondition( async () => { const {data: buildData} = await performApiRequest(`/translations/builds/${buildId}`); switch (buildData.status) { case BUILD_STATUS.finished: return true; case BUILD_STATUS.inProgress: case BUILD_STATUS.created: return false; default: throw new Error(`The translations build got an unexpected status '${buildData.status}'`); } }, { waitMs: BUILD_TIMEOUT_MS, intervalMs: 1000, }, ); const {data: downloadData} = await performApiRequest(`/translations/builds/${buildId}/download`); log.info(`Downloading translations to '${dstPath}'`); await net.downloadFile(downloadData.url, dstPath); } /** * * @param {string} srcDir * @param {string} dstDir * @returns {Promise} */ async function syncTranslatedDocuments(srcDir, dstDir) { const srcTranslatedDocs = (await walk(srcDir, DOCUMENTS_EXT)).map((p) => path.relative(srcDir, p)); if (srcTranslatedDocs.length === 0) { return; } let count = 0; for (const relativePath of srcTranslatedDocs) { log.info(`Synchronizing '${relativePath}' (${++count} of ${srcTranslatedDocs.length})`); await fs.mv( path.join(srcDir, relativePath), path.join(dstDir, relativePath), {mkdirp: true} ); } const dstDocs = (await walk(dstDir, DOCUMENTS_EXT)).map((p) => path.relative(dstDir, p)); const obsoleteDocs = _.difference(dstDocs, srcTranslatedDocs); for (const relativePath of obsoleteDocs) { log.info(`Removing the obsolete document '${relativePath}'`); await fs.rimraf(path.join(dstDir, relativePath)); } } /** * * @param {string} yamlPath * @returns {Promise} */ async function validateYaml(yamlPath) { log.debug(`Checking if '${yamlPath}' is a valid YAML file`); const validatorProcess = spawn('ruby', ['-ryaml', '-e', 'p YAML.load(STDIN.read)']); validatorProcess.stderr.on('data', (chunk) => log.debug(`STDERR: ${chunk.toString()}`)); validatorProcess.stdout.on('data', (chunk) => log.debug(`STDOUT: ${chunk.toString()}`)); const inputStream = fs.createReadStream(yamlPath); inputStream.pipe(validatorProcess.stdin); try { return await new Promise((resolve, reject) => { inputStream.once('error', reject); validatorProcess.once('exit', (code, signal) => { inputStream.unpipe(validatorProcess.stdin); log.debug(`Yaml validator process exited with code ${code}, signal ${signal}`); if (code === 0) { log.info(`'${yamlPath}' is a valid YAML file`); resolve(true); } else { reject(new Error(`'${yamlPath}' is not valid YAML file. Was it corrupted during translation?`)); } }); validatorProcess.once('error', (e) => { inputStream.unpipe(validatorProcess.stdin); reject(e); }); }); } catch (err) { validatorProcess.kill(9); log.warn(err.message); return false; } finally { validatorProcess.removeAllListeners(); inputStream.removeAllListeners(); } } /** * * @param {string} srcDir * @param {string} dstDir * @param {string} dstLanguage * @returns {Promise} */ async function syncTranslatedConfig(srcDir, dstDir, dstLanguage) { const srcPath = path.join(srcDir, CROWIN_MKDOCS_CONFIG); if (!await fs.exists(srcPath)) { throw new Error(`Did not find the translated MkDocs config at '${srcPath}'`); } const configFileName = ORIGINAL_MKDOCS_CONFIG(dstLanguage); if (!await validateYaml(srcPath)) { log.warn(`Skipping the corrupted translated MkDocs config '${configFileName}'`); return; } const dstPath = path.join(dstDir, configFileName); log.info(`Synchronizing '${dstPath}'`); await fs.mv(srcPath, dstPath); } async function main() { const buildId = await buildTranslations(); const zipPath = await tempDir.path({prefix: 'translations', suffix: '.zip'}); try { await downloadTranslations(buildId, zipPath); const tmpRoot = await tempDir.openDir(); try { await zip.extractAllTo(zipPath, tmpRoot); const srcLanguageNames = await fs.readdir(tmpRoot); log.info(`Available Crowdin languages: ${srcLanguageNames}`); log.info(`Supported languages map: ${JSON.stringify(CROWDIN_TO_FS_LANGUAGES_MAP)}`); let count = 0; for (const name of srcLanguageNames) { const currentPath = path.join(tmpRoot, name); if (!(await fs.stat(currentPath)).isDirectory() || name === DEFAULT_LANGUAGE) { continue; } const dstLanguageName = CROWDIN_TO_FS_LANGUAGES_MAP[name]; if (!dstLanguageName) { // If the target language is not present in the map we ignore it continue; } await syncTranslatedDocuments(currentPath, path.join(RESOURCES_ROOT, dstLanguageName)); await syncTranslatedConfig(currentPath, RESOURCES_ROOT, dstLanguageName); log.info( `Successfully updated resources for the '${dstLanguageName}' ` + `('${name}' in Crowdin) language (${++count} of ${Object.keys(CROWDIN_TO_FS_LANGUAGES_MAP).length})` ); } } finally { await fs.rimraf(tmpRoot); } } finally { await fs.rimraf(zipPath); } } (async () => await main())();