import path from 'node:path'; import stream from 'node:stream'; import {text} from 'node:stream/consumers'; import {pipeline} from 'node:stream/promises'; import {promisify} from 'node:util'; // @ts-ignore - archiver 8.0 types are not available; 7.x types don't match import {ZipArchive} from 'archiver'; import {exec} from 'teen_process'; import * as yauzl from 'yauzl'; import {fs} from './fs'; import {createBase64EncodeStream} from './internal'; import log from './logger'; import {isWindows} from './system'; import {Timer} from './timing'; import {GiB, isSubPath, memoize, toReadableSizeString} from './util'; const openZip = promisify(yauzl.open) as (zipPath: string, options?: yauzl.Options) => Promise; const ZIP_MAGIC = 'PK'; const IFMT = 0b1111000000000000; const IFDIR = 0b0100000000000000; const IFLNK = 0b1010000000000000; // Limit traversal to stop symlink cycles. const MAX_SYMLINK_TRAVERSALS = 40; export interface ExtractAllOptions { /** * The encoding to use for extracted file names. * For ZIP archives created on MacOS it is usually expected to be `utf8`. * By default it is autodetected based on the entry metadata and is only needed to be set explicitly * if the particular archive does not comply to the standards, which leads to corrupted file names * after extraction. Only applicable if system unzip binary is NOT being used. */ fileNamesEncoding?: BufferEncoding; /** * If true, attempt to use system unzip; if this fails, * fallback to the JS unzip implementation. */ useSystemUnzip?: boolean; } export interface ZipEntry { /** The actual entry instance */ entry: yauzl.Entry; /** * Async function which accepts the destination folder path * and extracts this entry into it. */ extractEntryTo: (destDir: string) => Promise; } export interface ZipOptions { /** Whether to encode the resulting archive to a base64-encoded string */ encodeToBase64?: boolean; /** Whether to log the actual archiver performance */ isMetered?: boolean; /** * The maximum size of the resulting archive in bytes. * This is set to 1GB by default, because Appium limits the maximum HTTP body size to 1GB. * Also, the NodeJS heap size must be enough to keep the resulting object * (usually this size is limited to 1.4 GB) */ maxSize?: number; /** * The compression level. * The maximum level is 9 (the best compression, worst performance). * The minimum compression level is 0 (no compression). */ level?: number; } export interface ZipCompressionOptions { /** * Compression level in range 0..9 * (greater numbers mean better compression, but longer processing time) */ level?: number; } export interface ZipSourceOptions { /** GLOB pattern for compression */ pattern?: string; /** The source root folder (the parent folder of the destination file by default) */ cwd?: string; /** The list of ignored patterns */ ignore?: string[]; } interface ZipExtractorOptions { dir: string; fileNamesEncoding?: BufferEncoding; defaultDirMode?: string; defaultFileMode?: string; } type OpenReadStreamFn = (entry: yauzl.Entry) => Promise; interface SymlinkTarget { linkTarget: string; linkParentDir: string; fileName: string; missingPaths: Set; } // This class is mostly copied from https://github.com/maxogden/extract-zip/blob/master/index.js class ZipExtractor { zipfile!: yauzl.ZipFile; private openReadStream!: OpenReadStreamFn; private readonly zipPath: string; private readonly opts: ZipExtractorOptions; private canceled = false; // Normalized missing-path keys -> links to recheck when those paths become symlinks. private readonly symlinkDependents = new Map>(); constructor(sourcePath: string, opts: ZipExtractorOptions) { this.zipPath = sourcePath; this.opts = opts; } async extract(): Promise { const {fileNamesEncoding} = this.opts; this.zipfile = await openZip(this.zipPath, { lazyEntries: true, // https://github.com/thejoshwolfe/yauzl/commit/cc7455ac789ba84973184e5ebde0581cdc4c3b39#diff-04c6e90faac2675aa89e2176d2eec7d8R95 decodeStrings: !fileNamesEncoding, }); this.openReadStream = createOpenReadStream(this.zipfile); this.canceled = false; const {dir} = this.opts; try { await processYauzlEntriesSequentially(this.zipfile, async (entry) => { if (this.canceled) { return false; } const fileName = this.extractFileName(entry); if (fileName.startsWith('__MACOSX/')) { return; } const dest = path.resolve(dir, fileName); if (!isContainedPath(dest, dir)) { throw new Error(`Out of bound path "${dest}" found while processing file ${fileName}`); } await this.extractEntry(entry, fileName, dest); }); } catch (err) { this.canceled = true; throw err; } } private extractFileName(entry: yauzl.Entry): string { return toEntryFileName(entry, this.opts.fileNamesEncoding); } private async extractEntry(entry: yauzl.Entry, fileName: string, dest: string): Promise { if (this.canceled) { return; } // convert external file attr int into a fs stat mode int const mode = (entry.externalFileAttributes >> 16) & 0xffff; // check if it's a symlink or dir (using stat mode constants) const isSymlink = (mode & IFMT) === IFLNK; const isDir = (mode & IFMT) === IFDIR || // Failsafe, borrowed from jsZip fileName.endsWith('/') || // check for windows weird way of specifying a directory // https://github.com/maxogden/extract-zip/issues/13#issuecomment-154494566 (entry.versionMadeBy >> 8 === 0 && entry.externalFileAttributes === 16); const procMode = this.getExtractedMode(mode, isDir) & 0o777; // always ensure folders are created const destDir = isDir ? dest : path.dirname(dest); const {dir} = this.opts; const realDestDir = await ensureDirWithinRoot(dir, destDir, fileName, isDir ? procMode : undefined); if (isDir) { return; } if (isSymlink) { const linkTarget = await text(await this.openReadStream(entry)); await this.createValidatedSymlink(linkTarget, dest, realDestDir, fileName); } else { await assertFileDestinationWithinRoot(dir, dest, realDestDir, fileName); const readStream = await this.openReadStream(entry); try { await writeFileNoFollow(readStream, dest, procMode); } finally { // Opening the output may fail before pipeline takes ownership of the input. readStream.destroy(); } } } private getExtractedMode(entryMode: number, isDir: boolean): number { const {defaultDirMode, defaultFileMode} = this.opts; let mode = entryMode; // Set defaults, if necessary if (mode === 0) { if (isDir) { if (defaultDirMode) { mode = parseInt(defaultDirMode, 10); } if (!mode) { mode = 0o755; } } else { if (defaultFileMode) { mode = parseInt(defaultFileMode, 10); } if (!mode) { mode = 0o644; } } } return mode; } /** * Reject escaping targets or excessive traversal; return missing-path keys for later rechecks. */ private async validateSymlinkTarget( linkTarget: string, linkParentDir: string, fileName: string, ): Promise> { // Windows resolves drive-relative targets (`C:foo`) against the drive's CWD, not the link's parent if (isWindows() && /^[A-Za-z]:/.test(linkTarget) && !path.win32.isAbsolute(linkTarget)) { throw new Error(`Out of bound symlink target "${linkTarget}" found while processing file ${fileName}`); } // Resolve links before '..'; realpath alone cannot handle missing targets. const separator = isWindows() ? /[\\/]+/ : /\/+/; const targetRoot = path.isAbsolute(linkTarget) ? path.parse(linkTarget).root : ''; let resolvedPath = targetRoot || linkParentDir; const remainingSegments = linkTarget.slice(targetRoot.length).split(separator); const missingPaths = new Set(); let followedLinks = 0; while (remainingSegments.length) { const segment = remainingSegments.shift(); if (!segment || segment === '.') { continue; } if (segment === '..') { resolvedPath = path.dirname(resolvedPath); continue; } const candidatePath = path.join(resolvedPath, segment); let nestedLinkTarget: string | null = null; try { const stats = await fs.lstat(candidatePath); if (stats.isSymbolicLink()) { nestedLinkTarget = await fs.readlink(candidatePath); } } catch (err: any) { if (err.code !== 'ENOENT') { throw err; } // An absent component may become a symlink in a later ZIP entry. missingPaths.add(this.getSymlinkDependencyKey(candidatePath)); } // Advance past ordinary or missing components. if (nestedLinkTarget === null) { resolvedPath = candidatePath; continue; } followedLinks += 1; if (followedLinks > MAX_SYMLINK_TRAVERSALS) { throw new Error(`Too many symlinks while processing file ${fileName}`); } // Resolve relative targets from the link's parent, absolute targets from their root. if (path.isAbsolute(nestedLinkTarget)) { resolvedPath = path.parse(nestedLinkTarget).root; nestedLinkTarget = nestedLinkTarget.slice(resolvedPath.length); } remainingSegments.unshift(...nestedLinkTarget.split(separator)); } if (!isContainedPath(resolvedPath, this.opts.dir)) { throw new Error(`Out of bound symlink target "${linkTarget}" found while processing file ${fileName}`); } return missingPaths; } private updateSymlinkDependencies(target: SymlinkTarget, missingPaths: Set): void { for (const missingPath of target.missingPaths) { const dependents = this.symlinkDependents.get(missingPath); dependents?.delete(target); if (!dependents?.size) { this.symlinkDependents.delete(missingPath); } } target.missingPaths = missingPaths; for (const missingPath of missingPaths) { let dependents = this.symlinkDependents.get(missingPath); if (!dependents) { dependents = new Set(); this.symlinkDependents.set(missingPath, dependents); } dependents.add(target); } } private async createValidatedSymlink( linkTarget: string, dest: string, linkParentDir: string, fileName: string, ): Promise { // Reject an escaping target before creating the link on disk. await this.validateSymlinkTarget(linkTarget, linkParentDir, fileName); await fs.symlink(linkTarget, dest); try { const target: SymlinkTarget = {linkTarget, linkParentDir, fileName, missingPaths: new Set()}; // Recheck for cycles and redirected forward references after creating the link. const key = this.getSymlinkDependencyKey(path.join(linkParentDir, path.basename(dest))); const affectedTargets = [target, ...(this.symlinkDependents.get(key) ?? [])]; for (const affectedTarget of affectedTargets) { const missingPaths = await this.validateSymlinkTarget( affectedTarget.linkTarget, affectedTarget.linkParentDir, affectedTarget.fileName, ); this.updateSymlinkDependencies(affectedTarget, missingPaths); } } catch (err) { // Roll back the link that invalidated a target. await fs.unlink(dest); throw err; } } private getSymlinkDependencyKey(absolutePath: string): string { // Normalize keys, not filesystem paths. Collisions only cause extra checks. return absolutePath.normalize('NFD').toLowerCase(); } } async function ensureDirWithinRoot(dir: string, dirPath: string, fileName: string, mode?: number): Promise { if (!isContainedPath(dirPath, dir)) { throw new Error(`Out of bound path "${dirPath}" found while processing file ${fileName}`); } const relativePath = path.relative(dir, dirPath); if (!relativePath) { return dir; } let currentPath = dir; for (const segment of relativePath.split(path.sep)) { currentPath = path.join(currentPath, segment); try { const [stats, realPath] = await Promise.all([fs.lstat(currentPath), fs.realpath(currentPath)]); if (!isContainedPath(realPath, dir)) { throw new Error(`Out of bound path "${currentPath}" found while processing file ${fileName}`); } if (!stats.isDirectory() && !stats.isSymbolicLink()) { throw new Error(`Cannot create a directory over existing file "${currentPath}"`); } } catch (err: any) { if (err.code !== 'ENOENT') { throw err; } await fs.mkdir(currentPath, {mode}); } } const realPath = await fs.realpath(dirPath); if (!isContainedPath(realPath, dir)) { throw new Error(`Out of bound path "${dirPath}" found while processing file ${fileName}`); } return realPath; } async function assertFileDestinationWithinRoot( dir: string, dest: string, realDestDir: string, fileName: string, ): Promise { let realDest: string | null = null; try { if ((await fs.lstat(dest)).isSymbolicLink()) { throw new Error(`Cannot write a file over existing symlink "${dest}"`); } realDest = await fs.realpath(dest); } catch (err: any) { if (err.code !== 'ENOENT') { throw err; } } realDest = realDest ?? path.resolve(realDestDir, path.basename(dest)); if (!isContainedPath(realDest, dir)) { throw new Error(`Out of bound path "${dest}" found while processing file ${fileName}`); } } async function writeFileNoFollow(readStream: stream.Readable, dest: string, mode: number): Promise { const flags = fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_TRUNC | (isWindows() ? 0 : fs.constants.O_NOFOLLOW); let file; try { file = await fs.openFile(dest, flags, mode); } catch (err) { // pipeline has not taken ownership of the input yet readStream.destroy(); throw err; } try { await pipeline(readStream, file.createWriteStream()); } finally { await file.close(); } } /** * Extract zipfile to a directory * * @param zipFilePath The full path to the source ZIP file * @param destDir The full path to the destination folder * @param opts Extraction options */ export async function extractAllTo(zipFilePath: string, destDir: string, opts: ExtractAllOptions = {}): Promise { if (!path.isAbsolute(destDir)) { throw new Error(`Target path '${destDir}' is expected to be absolute`); } await fs.mkdir(destDir, {recursive: true}); const dir = await fs.realpath(destDir); if (opts.useSystemUnzip) { try { await extractWithSystemUnzip(zipFilePath, dir); return; } catch (err: any) { log.warn('unzip failed; falling back to JS: %s', err.stderr || err.message); } } const extractor = new ZipExtractor(zipFilePath, { ...(opts as ExtractAllOptions & Partial), dir, }); await extractor.extract(); } /** * Extract a single zip entry to a directory * * @private * @param zipFile The source ZIP stream * @param entry The entry instance * @param destDir The full path to the destination folder * @param openReadStream Reused bound opener (one per archive) */ export async function _extractEntryTo( zipFile: yauzl.ZipFile, entry: yauzl.Entry, destDir: string, openReadStream: OpenReadStreamFn = createOpenReadStream(zipFile), ): Promise { const fileName = toEntryFileName(entry); const dstPath = path.resolve(destDir, fileName); if (!isContainedPath(dstPath, destDir)) { throw new Error(`Out of bound path "${dstPath}" found while processing file ${fileName}`); } // Same symlink-safe directory/file handling as in ZipExtractor await fs.mkdirp(destDir); const dir = await fs.realpath(destDir); const dest = path.resolve(dir, path.relative(destDir, dstPath)); if (fileName.endsWith('/')) { await ensureDirWithinRoot(dir, dest, fileName); return; } const realDestDir = await ensureDirWithinRoot(dir, path.dirname(dest), fileName); await assertFileDestinationWithinRoot(dir, dest, realDestDir, fileName); const readStream = await openReadStream(entry); await writeFileNoFollow(readStream, dest, 0o666); } /** * Get entries for a zip folder * * @param zipFilePath The full path to the source ZIP file * @param onEntry Callback when entry is read. * The callback is expected to accept one argument of ZipEntry type. * The iteration through the source zip file will be terminated as soon as * the result of this function equals to `false`. */ export async function readEntries( zipFilePath: string, onEntry: (entry: ZipEntry) => boolean | void | Promise, ): Promise { const zipfile = await openZip(zipFilePath, {lazyEntries: true}); const openReadStream = createOpenReadStream(zipfile); await processYauzlEntriesSequentially(zipfile, async (entry) => { const res = await onEntry({ entry, extractEntryTo: async (destDir: string) => await _extractEntryTo(zipfile, entry, destDir, openReadStream), }); return res === false ? false : undefined; }); } /** * Converts contents of local directory to an in-memory .zip buffer * * @param srcPath The full path to the folder or file being zipped * @param opts Zipping options * @returns Zipped (and encoded if `encodeToBase64` is truthy) * content of the source path as memory buffer * @throws {Error} if there was an error while reading the source * or the source is too big */ export async function toInMemoryZip(srcPath: string, opts: ZipOptions = {}): Promise { if (!(await fs.exists(srcPath))) { throw new Error(`No such file or folder: ${srcPath}`); } const {isMetered = true, encodeToBase64 = false, maxSize = 1 * GiB, level = 9} = opts; const resultBuffers: Buffer[] = []; let resultBuffersSize = 0; // Create a writable stream that zip buffers will be streamed to const resultWriteStream = new stream.Writable({ write(buffer: Buffer, _encoding: string, next: (err?: Error) => void) { resultBuffers.push(buffer); resultBuffersSize += buffer.length; if (maxSize > 0 && resultBuffersSize > maxSize) { resultWriteStream.emit( 'error', new Error(`The size of the resulting ` + `archive must not be greater than ${toReadableSizeString(maxSize)}`), ); } next(); }, }); // Zip 'srcDir' and stream it to the above writable stream const archive = new ZipArchive({ zlib: {level}, }); let srcSize: number | null = null; const base64EncoderStream = encodeToBase64 ? createBase64EncodeStream() : null; const resultWriteStreamPromise = new Promise((resolve, reject) => { resultWriteStream.once('error', (e: Error) => { if (base64EncoderStream) { archive.unpipe(base64EncoderStream); base64EncoderStream.unpipe(resultWriteStream); } else { archive.unpipe(resultWriteStream); } archive.abort(); archive.destroy(); reject(e); }); resultWriteStream.once('finish', () => { srcSize = archive.pointer(); resolve(); }); }); const archiveStreamPromise = new Promise((resolve, reject) => { archive.once('finish', resolve); archive.once('error', (e: Error) => reject(new Error(`Failed to archive '${srcPath}': ${e.message}`))); }); const timer = isMetered ? new Timer().start() : null; if ((await fs.stat(srcPath)).isDirectory()) { archive.directory(srcPath, false); } else { archive.file(srcPath, { name: path.basename(srcPath), }); } if (base64EncoderStream) { archive.pipe(base64EncoderStream); base64EncoderStream.pipe(resultWriteStream); } else { archive.pipe(resultWriteStream); } await Promise.all([archive.finalize(), archiveStreamPromise, resultWriteStreamPromise]); if (timer) { log.debug( `Zipped ${encodeToBase64 ? 'and base64-encoded ' : ''}` + `'${path.basename(srcPath)}' ` + (srcSize ? `(${toReadableSizeString(srcSize)}) ` : '') + `in ${timer.getDuration().asSeconds.toFixed(3)}s ` + `(compression level: ${level})`, ); } // Return the array of zip buffers concatenated into one buffer return Buffer.concat(resultBuffers); } /** * Verifies whether the given file is a valid ZIP archive * * @param filePath - Full path to the file * @throws {Error} If the file does not exist or is not a valid ZIP archive */ export async function assertValidZip(filePath: string): Promise { if (!(await fs.exists(filePath))) { throw new Error(`The file at '${filePath}' does not exist`); } const {size} = await fs.stat(filePath); if (size < 4) { throw new Error(`The file at '${filePath}' is too small to be a ZIP archive`); } const fd = await fs.open(filePath, 'r'); try { const buffer = Buffer.alloc(ZIP_MAGIC.length); await fs.read(fd, buffer, 0, ZIP_MAGIC.length, 0); const signature = buffer.toString('ascii'); if (signature !== ZIP_MAGIC) { throw new Error( `The file signature '${signature}' of '${filePath}' ` + `is not equal to the expected ZIP archive signature '${ZIP_MAGIC}'`, ); } return true; } finally { await fs.close(fd); } } /** * Creates an archive based on the given glob pattern * * @param dstPath - The resulting archive path * @param src - Source options * @param opts - Compression options * @throws {Error} If there was an error while creating the archive */ export async function toArchive( dstPath: string, src: ZipSourceOptions = {}, opts: ZipCompressionOptions = {}, ): Promise { const {level = 9} = opts; const {pattern = '**/*', cwd = path.dirname(dstPath), ignore = []} = src; const archive = new ZipArchive({zlib: {level}}); const outStream = fs.createWriteStream(dstPath); await new Promise((resolve, reject) => { const outFinished = new Promise((_resolve, _reject) => { outStream.once('error', (e: Error) => { archive.unpipe(outStream); archive.abort(); archive.destroy(); _reject(e); }); outStream.once('finish', () => _resolve()); }); archive .glob(pattern, { cwd, ignore, }) .on('error', reject) .pipe(outStream); void Promise.all([archive.finalize(), outFinished]) .then(() => resolve()) .catch(reject); }); } function createOpenReadStream(zipfile: yauzl.ZipFile): OpenReadStreamFn { return promisify(zipfile.openReadStream.bind(zipfile)) as OpenReadStreamFn; } function toEntryFileName(entry: yauzl.Entry, encoding?: BufferEncoding): string { if (Buffer.isBuffer(entry.fileName)) { return entry.fileName.toString(encoding ?? 'utf8'); } return entry.fileName; } /** * Walk zip entries sequentially (yauzl `lazyEntries` contract). */ async function processYauzlEntriesSequentially( zipfile: yauzl.ZipFile, onEntry: (entry: yauzl.Entry) => Promise, ): Promise { let queue: Promise = Promise.resolve(); let stopped = false; await new Promise((resolve, reject) => { let settled = false; const settleError = (err: unknown) => { if (!settled) { settled = true; reject(err); } }; const settleSuccess = async () => { try { await queue; if (!settled) { settled = true; resolve(); } } catch (err) { settleError(err); } }; zipfile.on('error', settleError); zipfile.on('end', () => void settleSuccess()); zipfile.on('close', () => void settleSuccess()); zipfile.on('entry', (entry: yauzl.Entry) => { queue = queue .then(async () => { if (stopped) { return undefined; } const shouldContinue = await onEntry(entry); if (shouldContinue === false) { stopped = true; zipfile.close(); return undefined; } zipfile.readEntry(); return undefined; }) .catch((err: unknown) => { stopped = true; zipfile.close(); settleError(err); return undefined; }); }); zipfile.readEntry(); }); } /** * Executes system unzip (e.g., `/usr/bin/unzip`). If available, it is * significantly faster than the JS implementation. * By default all files in the destDir get overridden if already exist. * * @param zipFilePath The full path to the source ZIP file * @param destDir The full path to the destination folder. * This folder is expected to already exist before extracting the archive. */ async function extractWithSystemUnzip(zipFilePath: string, destDir: string): Promise { const isWindowsHost = isWindows(); let executablePath: string; try { executablePath = await getExecutablePath(isWindowsHost ? 'powershell.exe' : 'unzip'); } catch (e) { throw new Error('Could not find system unzip', {cause: e}); } if (isWindowsHost) { // on Windows we use PowerShell to unzip files await exec(executablePath, [ '-command', 'Expand-Archive', '-LiteralPath', zipFilePath, '-DestinationPath', destDir, '-Force', ]); } else { // -q means quiet (no stdout) // -o means overwrite // -d is the dest dir await exec(executablePath, ['-q', '-o', zipFilePath, '-d', destDir]); } } /** * Finds and memoizes the full path to the given executable. * Rejects if it is not found. */ const getExecutablePath = memoize( /** * @returns Full Path to the executable */ async function getExecutablePath(binaryName: string): Promise { const fullPath = await fs.which(binaryName); log.debug(`Found '${binaryName}' at '${fullPath}'`); return fullPath; }, ); function isContainedPath(originalPath: string, root: string): boolean { // `startsWith('..')` would reject a safe name like `..foo`; isSubPath only treats `..` as a parent return isSubPath(originalPath, root); } export default { extractAllTo, readEntries, toInMemoryZip, assertValidZip, toArchive, };