appium/packages/base-driver/lib/jsonwp-proxy/proxy.ts

import http from 'node:http';
import https from 'node:https';
import nodeUrl from 'node:url';

import {logger, util} from '@appium/support';
import type {AppiumLogger, HTTPBody, HTTPHeaders, HTTPMethod, ProxyOptions, ProxyResponse} from '@appium/types';
import type {AxiosError, AxiosResponse, RawAxiosRequestConfig} from 'axios';
import type {Request, Response} from 'express';
import {match as pathToRegexMatch} from 'path-to-regexp';

import {DEFAULT_BASE_PATH, MAX_LOG_BODY_LENGTH, PROTOCOLS} from '../constants';
import {getSummaryByCode} from '../jsonwp-status/status';
import {isSessionCommand, routeToCommandName} from '../protocol';
import {
  errorFromMJSONWPStatusCode,
  errorFromW3CJsonCode,
  errors,
  getResponseForW3CError,
  isErrorType,
} from '../protocol/errors';
import {ensureW3cResponse, formatResponseValue} from '../protocol/helpers';
import {omit, pick} from '../utils';
import {ProtocolConverter} from './protocol-converter';
import {ProxyRequest} from './proxy-request';

const DEFAULT_LOG = logger.getLogger('WD Proxy');
const DEFAULT_REQUEST_TIMEOUT = 240000;
const COMMAND_WITH_SESSION_ID_MATCHER = pathToRegexMatch('{/*prefix}/session/:sessionId{/*command}');

const {MJSONWP, W3C} = PROTOCOLS;

type Protocol = (typeof PROTOCOLS)[keyof typeof PROTOCOLS];

const ALLOWED_OPTS = [
  'scheme',
  'server',
  'port',
  'base',
  'reqBasePath',
  'sessionId',
  'timeout',
  'log',
  'keepAlive',
  'headers',
] as const;

export class JWProxy {
  readonly scheme: string;
  readonly server: string;
  readonly port: number;
  readonly base: string;
  readonly reqBasePath: string;
  sessionId: string | null;
  timeout: number;
  readonly headers: HTTPHeaders | undefined;
  readonly httpAgent: http.Agent;
  readonly httpsAgent: https.Agent;
  readonly protocolConverter: ProtocolConverter;

  private _downstreamProtocol: Protocol | null | undefined;
  private _activeRequests: ProxyRequest[];
  private readonly _log: AppiumLogger | undefined;

  constructor(opts: ProxyOptions = {}) {
    const filteredOptsWithoutLog = omit(pick(opts as Record<string, unknown>, ALLOWED_OPTS), 'log') as Omit<
      ProxyOptions,
      'log'
    >;
    const options = {
      scheme: 'http',
      server: 'localhost',
      port: 4444,
      base: DEFAULT_BASE_PATH,
      reqBasePath: DEFAULT_BASE_PATH,
      sessionId: null,
      timeout: DEFAULT_REQUEST_TIMEOUT,
      ...filteredOptsWithoutLog,
    } as ProxyOptions & {
      scheme: string;
      server: string;
      port: number;
      base: string;
      reqBasePath: string;
      sessionId: string | null;
      timeout: number;
    };
    options.scheme = options.scheme.toLowerCase();
    this.scheme = options.scheme;
    this.server = options.server;
    this.port = options.port;
    this.base = options.base;
    this.reqBasePath = options.reqBasePath;
    this.sessionId = options.sessionId;
    this.timeout = options.timeout;
    this.headers = options.headers;

    this._activeRequests = [];
    this._downstreamProtocol = null;
    const agentOpts = {
      keepAlive: opts.keepAlive ?? true,
      maxSockets: 10,
      maxFreeSockets: 5,
    };
    this.httpAgent = new http.Agent(agentOpts);
    this.httpsAgent = new https.Agent(agentOpts);
    this.protocolConverter = new ProtocolConverter(this.proxy.bind(this), opts.log);
    this._log = opts.log;

    this.log.debug(`${this.constructor.name} options: ${JSON.stringify(options)}`);
  }

  get log(): AppiumLogger {
    return this._log ?? DEFAULT_LOG;
  }

  /**
   * Gets the protocol used by the downstream server (W3C or MJSONWP).
   */
  get downstreamProtocol(): Protocol | null | undefined {
    return this._downstreamProtocol;
  }

  /**
   * Sets the protocol used by the downstream server (W3C or MJSONWP).
   */
  set downstreamProtocol(value: Protocol | null | undefined) {
    this._downstreamProtocol = value;
    this.protocolConverter.downstreamProtocol = value;
  }

  /**
   * Returns the number of active downstream HTTP requests.
   */
  getActiveRequestsCount(): number {
    return this._activeRequests.length;
  }

  /**
   * Cancels all currently active downstream HTTP requests.
   */
  cancelActiveRequests(): void {
    for (const ar of this._activeRequests) {
      ar.cancel();
    }
    this._activeRequests = [];
  }

  /**
   * Builds a full downstream URL (including base path and session) for a given upstream URL.
   */
  getUrlForProxy(url: string, method?: HTTPMethod): string {
    const parsedUrl = this._parseUrl(url);
    const normalizedPathname = this._toNormalizedPathname(parsedUrl);
    const commandName = normalizedPathname ? routeToCommandName(normalizedPathname, method) : '';
    const requiresSessionId = !commandName || (commandName && isSessionCommand(commandName));
    const proxyPrefix = `${this.scheme}://${this.server}:${this.port}${this.base}`;
    let proxySuffix = normalizedPathname ? `/${normalizedPathname.replace(/^\/+/, '')}` : '';
    if (parsedUrl.search) {
      proxySuffix += parsedUrl.search;
    }
    if (!requiresSessionId) {
      return `${proxyPrefix}${proxySuffix}`;
    }
    if (!this.sessionId) {
      throw new ReferenceError(`Session ID is not set, but saw a URL that requires it (${url})`);
    }
    return `${proxyPrefix}/session/${this.sessionId}${proxySuffix}`;
  }

  /**
   * Proxies a raw WebDriver command to the downstream server.
   */
  async proxy(url: string, method: string, body: HTTPBody = null): Promise<[ProxyResponse, HTTPBody]> {
    method = method.toUpperCase();
    const newUrl = this.getUrlForProxy(url, method as HTTPMethod);
    const truncateBody = (content: unknown): string =>
      util.truncateString(typeof content === 'string' ? content : JSON.stringify(content), {
        length: MAX_LOG_BODY_LENGTH,
      });
    const reqOpts: RawAxiosRequestConfig = {
      url: newUrl,
      method,
      headers: {
        'content-type': 'application/json; charset=utf-8',
        'user-agent': 'appium',
        accept: 'application/json, */*',
        ...(this.headers ?? {}),
      },
      proxy: false,
      timeout: this.timeout,
      httpAgent: this.httpAgent,
      httpsAgent: this.httpsAgent,
    };
    // GET methods shouldn't have any body. Most servers are OK with this,
    // but WebDriverAgent throws 400 errors
    if (util.hasValue(body) && method !== 'GET') {
      if (typeof body !== 'object') {
        try {
          reqOpts.data = JSON.parse(body as string);
        } catch (error) {
          this.log.warn(
            'Invalid body payload (%s): %s',
            (error as Error).message,
            logger.markSensitive(truncateBody(body)),
          );
          throw new Error('Cannot interpret the request body as valid JSON. Check the server log for more details.', {
            cause: error,
          });
        }
      } else {
        reqOpts.data = body;
      }
    }

    this.log.debug(
      `Proxying [%s %s] to [%s %s] with ${reqOpts.data ? 'body: %s' : '%s body'}`,
      method,
      url || '/',
      method,
      newUrl,
      reqOpts.data ? logger.markSensitive(truncateBody(reqOpts.data)) : 'no',
    );

    const throwProxyError = (error: unknown): never => {
      const err = new Error(`The request to ${url} has failed`) as Error & {
        response: {data: unknown; status: number};
      };
      err.response = {
        data: error,
        status: 500,
      };
      throw err;
    };
    let isResponseLogged = false;
    try {
      const {data, status, headers} = await this.request(reqOpts);
      // `data` might be really big
      // Be careful while handling it to avoid memory leaks
      if (!util.isPlainObject(data)) {
        // The response should be a valid JSON object
        // If it cannot be coerced to an object then the response is wrong
        throwProxyError(data);
      }
      this.log.debug(`Got response with status ${status}: %s`, logger.markSensitive(truncateBody(data)));
      isResponseLogged = true;
      const isSessionCreationRequest = url.endsWith('/session') && method === 'POST';
      if (isSessionCreationRequest) {
        if (status === 200) {
          const value = data.value as Record<string, unknown> | undefined;
          const raw = data.sessionId ?? value?.sessionId;
          this.sessionId = typeof raw === 'string' ? raw : raw != null ? String(raw) : null;
        }
        this.downstreamProtocol = this.getProtocolFromResBody(data) ?? this.downstreamProtocol;
        this.log.info(`Determined the downstream protocol as '${this.downstreamProtocol}'`);
      }
      if (Object.hasOwn(data, 'status') && parseInt(data.status as string, 10) !== 0) {
        throwProxyError(data);
      }
      return [
        {
          statusCode: status,
          headers: headers as HTTPHeaders,
          body: data,
        },
        data,
      ];
    } catch (e: unknown) {
      const err = e as AxiosError<unknown> & {message: string};
      let proxyErrorMsg = err.message;
      if (util.hasValue(err.response)) {
        if (!isResponseLogged) {
          const error = logger.markSensitive(truncateBody(err.response.data));
          this.log.info(
            util.hasValue(err.response.status)
              ? `Got response with status ${err.response.status}: %s`
              : `Got response with unknown status: %s`,
            error,
          );
        }
      } else {
        proxyErrorMsg = `Could not proxy command to the remote server. Original error: ${err.message}`;
        this.log.info(err.message);
      }
      throw new errors.ProxyRequestError(proxyErrorMsg, err.response?.data, err.response?.status);
    }
  }

  /**
   * Detects the downstream protocol from a response body.
   */
  getProtocolFromResBody(resObj: Record<string, unknown>): Protocol | undefined {
    if (Number.isInteger(resObj.status)) {
      return MJSONWP;
    }
    if (resObj.value !== undefined) {
      return W3C;
    }
  }

  /**
   * Proxies a command identified by its HTTP method and URL to the downstream server.
   */
  async proxyCommand(url: string, method: HTTPMethod, body: HTTPBody = null): Promise<[ProxyResponse, HTTPBody]> {
    const parsedUrl = this._parseUrl(url);
    const normalizedPathname = this._toNormalizedPathname(parsedUrl);
    const commandName = normalizedPathname ? routeToCommandName(normalizedPathname, method) : '';
    if (!commandName) {
      return await this.proxy(url, method, body);
    }
    this.log.debug(`Matched '${url}' to command name '${commandName}'`);

    return await this.protocolConverter.convertAndProxy(commandName, url, method, body);
  }

  /**
   * Executes a WebDriver command and returns the unwrapped `value` field (or throws).
   */
  async command(url: string, method: HTTPMethod, body: HTTPBody = null): Promise<HTTPBody> {
    let response: ProxyResponse;
    let resBodyObj: HTTPBody;
    try {
      [response, resBodyObj] = await this.proxyCommand(url, method, body);
    } catch (err: unknown) {
      if (isErrorType(err, errors.ProxyRequestError)) {
        throw err.getActualError();
      }
      throw new errors.UnknownError((err as Error).message);
    }
    const resBody = resBodyObj as Record<string, unknown>;
    const protocol = this.getProtocolFromResBody(resBody);
    if (protocol === MJSONWP) {
      if (response.statusCode === 200 && resBody.status === 0) {
        return resBody.value;
      }
      const status = parseInt(resBody.status as string, 10);
      if (!isNaN(status) && status !== 0) {
        let message: unknown = resBody.value;
        if (util.isPlainObject(message) && Object.hasOwn(message, 'message')) {
          message = (message as Record<string, unknown>).message;
        }
        throw errorFromMJSONWPStatusCode(
          status,
          util.isEmpty(message) ? getSummaryByCode(status) : (message as string | {message: string}),
        );
      }
    } else if (protocol === W3C) {
      if (response.statusCode < 300) {
        return resBody.value;
      }
      if (util.isPlainObject(resBody.value) && resBody.value.error) {
        const value = resBody.value;
        throw errorFromW3CJsonCode(
          value.error as string,
          (value.message as string) ?? '',
          value.stacktrace as string | undefined,
        );
      }
    } else if (response.statusCode === 200) {
      return resBodyObj;
    }
    throw new errors.UnknownError(
      `Did not know what to do with response code '${response.statusCode}' ` +
        `and response body '${util.truncateString(JSON.stringify(resBodyObj), {
          length: 300,
        })}'`,
    );
  }

  /**
   * Extracts a session id from a WebDriver-style URL.
   */
  getSessionIdFromUrl(url: string): string | null {
    const match = url.match(/\/session\/([^/]+)/);
    return match ? match[1] : null;
  }

  /**
   * Proxies an Express `Request`/`Response` pair to the downstream server,
   * converting any downstream errors into a proper W3C HTTP response.
   *
   * This method must not throw; it always writes a response.
   */
  async proxyReqRes(req: Request, res: Response): Promise<void> {
    let statusCode: number;
    let resBodyObj: HTTPBody;
    try {
      const [response, body] = await this.proxyCommand(req.originalUrl, req.method as HTTPMethod, req.body);
      statusCode = response.statusCode;
      resBodyObj = body;
    } catch (err: unknown) {
      [statusCode, resBodyObj] = getResponseForW3CError(
        isErrorType(err, errors.ProxyRequestError)
          ? (err as InstanceType<typeof errors.ProxyRequestError>).getActualError()
          : err,
      );
    }
    res.setHeader('content-type', 'application/json; charset=utf-8');
    if (!util.isPlainObject(resBodyObj)) {
      const error = new errors.UnknownError(
        `The downstream server response with the status code ${statusCode} is not a valid JSON object: ` +
          util.truncateString(`${resBodyObj}`, {length: 300}),
      );
      [statusCode, resBodyObj] = getResponseForW3CError(error);
    }

    const resBody = resBodyObj as Record<string, unknown>;
    if (Object.hasOwn(resBody, 'sessionId')) {
      const reqSessionId = this.getSessionIdFromUrl(req.originalUrl);
      if (reqSessionId) {
        this.log.info(`Replacing sessionId ${resBody.sessionId} with ${reqSessionId}`);
        resBody.sessionId = reqSessionId;
      } else if (this.sessionId) {
        this.log.info(`Replacing sessionId ${resBody.sessionId} with ${this.sessionId}`);
        resBody.sessionId = this.sessionId;
      }
    }
    resBody.value = formatResponseValue(resBody.value as object | undefined);
    res.status(statusCode).json(ensureW3cResponse(resBody));
  }

  /**
   * Performs requests to the downstream server
   *
   * @private - Do not call this method directly,
   * it uses client-specific arguments and responses!
   */
  private async request(requestConfig: RawAxiosRequestConfig): Promise<AxiosResponse> {
    const req = new ProxyRequest(requestConfig);
    this._activeRequests.push(req);
    try {
      return await req.execute();
    } finally {
      const reqIndex = this._activeRequests.indexOf(req);
      if (reqIndex >= 0) {
        this._activeRequests.splice(reqIndex, 1);
      }
    }
  }

  private _parseUrl(url: string): nodeUrl.UrlWithStringQuery {
    // eslint-disable-next-line n/no-deprecated-api -- we need relative URL support
    const parsedUrl = nodeUrl.parse(url || '/');
    if (
      parsedUrl.href == null ||
      parsedUrl.pathname == null ||
      (parsedUrl.protocol && !['http:', 'https:'].includes(parsedUrl.protocol))
    ) {
      throw new Error(`Did not know how to proxy the url '${url}'`);
    }
    return parsedUrl;
  }

  private _toNormalizedPathname(parsedUrl: nodeUrl.UrlWithStringQuery): string {
    if (typeof parsedUrl.pathname !== 'string') {
      return '';
    }
    let pathname =
      this.reqBasePath && parsedUrl.pathname.startsWith(this.reqBasePath)
        ? parsedUrl.pathname.replace(this.reqBasePath, '')
        : parsedUrl.pathname;
    const match = COMMAND_WITH_SESSION_ID_MATCHER(pathname);
    // This is needed for the backward compatibility
    // if drivers don't set reqBasePath properly
    if (!this.reqBasePath) {
      if (match && match.params && Array.isArray((match.params as Record<string, unknown>).prefix)) {
        pathname = pathname.replace(`/${((match.params as Record<string, unknown>).prefix as string[]).join('/')}`, '');
      } else if (pathname.startsWith('/wd/hub')) {
        pathname = pathname.replace('/wd/hub', '');
      }
    }
    let result = pathname;
    if (match && match.params) {
      const command = (match.params as Record<string, unknown>).command;
      result = Array.isArray(command) ? `/${(command as string[]).join('/')}` : '';
    }
    return result.replace(/\/+$/, '');
  }
}