diff --git a/CHANGELOG.md b/CHANGELOG.md index cff9f381b..cdb9a6a6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -167,6 +167,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Editing a file that defines the same name more than once, like two classes that each have an `execute` method or a method's overloads, no longer moves every caller from other files onto one of them during `codegraph sync`. A sync interrupted partway through a file also no longer loses those callers until a full re-index. Thanks @ijbranch for the report. (#2276) - Delphi and Free Pascal include files (`.inc`) are now indexed as Pascal. Before, they were read as PHP and came up empty. A `.inc` file with a ` { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-fnref-global-')); + fs.writeFileSync(path.join(tmpDir, 'store.py'), 'class Store:\n def fetch(self, ids):\n return ids\n'); + fs.writeFileSync(path.join(tmpDir, 'decoy.py'), 'class Decoy:\n def fetch(self, ids):\n return []\n'); + fs.writeFileSync(path.join(tmpDir, 'settings.py'), `from decoy import Decoy +conn = None + +def init(): + global conn + from store import Store + conn = Store() + +def shadow(): + conn = Decoy() + return conn + +def same_file(pool): + pool.submit(conn.fetch, []) + +def param_shadow(pool, conn): + pool.submit(conn.fetch, []) + +def loop_shadow(pool, items): + for conn in items: + pool.submit(conn.fetch, []) +`); + fs.writeFileSync(path.join(tmpDir, 'consumer.py'), `import settings +from settings import conn + +def via_module(pool): + pool.submit(settings.conn.fetch, []) + +def via_name(pool): + pool.submit(conn.fetch, []) + +def import_shadow(pool, settings): + pool.submit(settings.conn.fetch, []) + +def local_import(pool): + from settings import conn + pool.submit(conn.fetch, []) +`); + const cg = CodeGraph.initSync(tmpDir); + try { + await cg.indexAll(); + const store = cg.getNodesByName('fetch').find(n => n.qualifiedName.startsWith('Store::'))!; + const into = cg.getIncomingEdges(store.id).filter(e => e.kind === 'references' && e.metadata?.fnRef === true); + // A parameter or loop variable of the same name is not the global; a local import of it is. + expect(sourceNames(cg, into)).toEqual(['local_import', 'same_file', 'via_module', 'via_name']); + const decoy = cg.getNodesByName('fetch').find(n => n.qualifiedName.startsWith('Decoy::'))!; + expect(cg.getIncomingEdges(decoy.id).filter(e => e.metadata?.fnRef === true)).toHaveLength(0); + } finally { cg.close(); } + }); + + it('#1820: a module global with several backends binds to the declaration they share', async () => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-fnref-global-many-')); + fs.mkdirSync(path.join(tmpDir, 'backends')); + fs.writeFileSync(path.join(tmpDir, 'backends', '__init__.py'), ''); + fs.writeFileSync(path.join(tmpDir, 'backends', 'base.py'), + 'class Base:\n def fetch(self, ids):\n raise NotImplementedError("backend")\n' + + ' def search(self):\n raise NotImplementedError("backend")\n'); + fs.writeFileSync(path.join(tmpDir, 'backends', 'es.py'), + 'from backends.base import Base\nclass ES(Base):\n def search(self):\n return []\n'); + fs.writeFileSync(path.join(tmpDir, 'backends', 'vec.py'), + 'from backends.base import (\n Base,\n)\nclass Vec(Base):\n def fetch(self, ids):\n return ids\n' + + ' def search(self):\n return []\n'); + fs.writeFileSync(path.join(tmpDir, 'other.py'), 'class Other:\n def fetch(self, ids):\n return ids\n'); + fs.writeFileSync(path.join(tmpDir, 'settings.py'), `import backends.es +conn = None + +def init(engine): + global conn + if engine == "es": + conn = backends.es.ES() + else: + from backends import vec as vec_module + conn = vec_module.Vec() +`); + fs.writeFileSync(path.join(tmpDir, 'unknown.py'), `from other import Other +conn = None +mixed = None + +def init(flag): + global conn, mixed + conn = make_conn() + mixed = Other() if flag else None +`); + fs.writeFileSync(path.join(tmpDir, 'consumer.py'), `import settings +import unknown + +def gc(pool): + pool.submit(settings.conn.fetch, []) + +def find(pool): + pool.submit(settings.conn.search) + +def opaque(pool): + pool.submit(unknown.conn.fetch, []) + pool.submit(unknown.mixed.fetch, []) +`); + const cg = CodeGraph.initSync(tmpDir); + try { + await cg.indexAll(); + const byOwner = (owner: string, name = 'fetch') => cg.getNodesByName(name).find(n => n.qualifiedName.startsWith(`${owner}::`))!; + const fnRefs = (owner: string, name = 'fetch') => cg.getIncomingEdges(byOwner(owner, name).id) + .filter(e => e.kind === 'references' && e.metadata?.fnRef === true); + // ES inherits Base.fetch and Vec overrides it: the shared declaration is Base.fetch. + expect(sourceNames(cg, fnRefs('Base'))).toEqual(['gc']); + expect(fnRefs('Vec')).toHaveLength(0); + // Both backends override search: the declaration they inherit is still Base.search. + expect(sourceNames(cg, fnRefs('Base', 'search'))).toEqual(['find']); + expect([...fnRefs('ES', 'search'), ...fnRefs('Vec', 'search')]).toHaveLength(0); + // An opaque initializer (`make_conn()`, a conditional) leaves the global's type unknown. + expect(fnRefs('Other')).toHaveLength(0); + } finally { cg.close(); } + }); + + // One index per adversarial case: several seconds on a loaded machine. + it('#1820: a module global never binds through a shadow, an unknown value or a deeper chain', async () => { + const header = `from store import Store +from decoy import Decoy +conn = None + +def init(): + global conn + conn = Store() +`; + // Each case: its files, and the fnRef edges into any `fetch` it must produce. + const cases: Record, string[]]> = { + control: [{ 'm.py': header + 'def cb(pool):\n pool.submit(conn.fetch)\n' }, ['cb -> Store::fetch']], + tuple_first: [{ 'm.py': header + 'def cb(pool):\n conn, _ = (Decoy(), None)\n pool.submit(conn.fetch)\n' }, []], + loop_and_nested_global: [{ 'm.py': header + + 'def cb(pool, items):\n for conn in items:\n pass\n def nested():\n global conn\n pool.submit(conn.fetch)\n' }, []], + multiline_conditional: [{ 'm.py': header.replace('conn = Store()', 'conn = Store(\n x=1,\n ) if flag else Decoy()') + + 'def cb(pool):\n pool.submit(conn.fetch)\n' }, []], + multiline_constructor: [{ 'm.py': header.replace('conn = Store()', 'conn = Store(\n x=1,\n )') + + 'def cb(pool):\n pool.submit(conn.fetch)\n' }, ['cb -> Store::fetch']], + closure_param: [{ 'm.py': header + 'def outer(conn):\n def cb(pool):\n pool.submit(conn.fetch)\n return cb\n' }, []], + closure_local: [{ 'm.py': header + 'def outer(pool):\n conn = Decoy()\n def cb():\n pool.submit(conn.fetch)\n return cb\n' }, []], + lambda_param: [{ 'm.py': header + 'def cb(pool):\n pool.submit(lambda conn: pool.map(conn.fetch))\n' }, []], + comprehension: [{ 'm.py': header + 'def cb(pool, xs):\n [pool.submit(conn.fetch) for conn in xs]\n' }, []], + except_as: [{ 'm.py': header + 'def cb(pool):\n try:\n pass\n except Exception as conn:\n pool.submit(conn.fetch)\n' }, []], + match_as: [{ 'm.py': header + 'def cb(pool, x):\n match x:\n case Decoy() as conn:\n pool.submit(conn.fetch)\n' }, []], + global_in_nested_def: [{ 'm.py': 'from decoy import Decoy\nconn = None\n\ndef init():\n def helper():\n global conn\n conn = Decoy()\n\n' + + 'def cb(pool):\n pool.submit(conn.fetch)\n' }, []], + keyword_in_multiline_call: [{ 'm.py': 'from decoy import Decoy\nconn = None\napp = dict(\n conn=Decoy()\n)\n\n' + + 'def cb(pool):\n pool.submit(conn.fetch)\n' }, []], + tuple_rebind: [{ 'm.py': header + 'def reset():\n global conn\n conn, other = Decoy(), 1\n\ndef cb(pool):\n pool.submit(conn.fetch)\n' }, []], + with_rebind: [{ 'm.py': header + 'def reset():\n global conn\n with open_decoy() as conn:\n pass\n\ndef cb(pool):\n pool.submit(conn.fetch)\n' }, []], + docstring: [{ 'm.py': 'from store import Store\nconn = None\n"""\nconn = Store()\n"""\n\ndef cb(pool):\n pool.submit(conn.fetch)\n' }, []], + deeper_chain: [{ 'settings.py': header, 'c.py': 'import settings\n\ndef cb(pool):\n pool.submit(settings.conn.pool.fetch)\n' }, []], + importer_rebinds: [{ 'settings.py': header, + 'c.py': 'from settings import conn\nfrom decoy import Decoy\n\ndef reset():\n global conn\n conn = Decoy()\n\ndef cb(pool):\n pool.submit(conn.fetch)\n' }, []], + local_import_of_other_module: [{ 'settings.py': header, 'other_settings.py': 'from decoy import Decoy\nconn = Decoy()\n', + 'c.py': 'import settings\n\ndef cb(pool):\n import other_settings as settings\n pool.submit(settings.conn.fetch)\n' }, []], + local_from_import_of_other_module: [{ 'settings.py': header, 'other_settings.py': 'from decoy import Decoy\nconn = Decoy()\n', + 'c.py': 'from settings import conn\n\ndef cb(pool):\n from other_settings import conn\n pool.submit(conn.fetch)\n' }, []], + comment_in_parenthesized_import: [{ 'settings.py': header + 'other = 1\n', + 'c.py': 'from settings import (\n other, # see fetch()\n conn,\n)\n\ndef cb(pool):\n pool.submit(conn.fetch)\n' }, ['cb -> Store::fetch']], + star_import_override: [{ 'local.py': 'from decoy import Decoy\nconn = Decoy()\n', 'settings.py': header + 'from local import *\n', + 'c.py': 'import settings\n\ndef cb(pool):\n pool.submit(settings.conn.fetch)\n' }, []], + match_sequence_capture: [{ 'm.py': header + 'def cb(pool, x):\n match x:\n case [conn]:\n pool.submit(conn.fetch)\n' }, []], + match_keyword_capture: [{ 'm.py': header + 'def cb(pool, x):\n match x:\n case Decoy(inner=conn):\n pool.submit(conn.fetch)\n' }, []], + match_bare_capture: [{ 'm.py': header + 'def cb(pool, x):\n match x:\n case conn:\n pool.submit(conn.fetch)\n' }, []], + multiline_loop_target: [{ 'm.py': header + 'def cb(pool, xs):\n for (a,\n conn) in xs:\n pool.submit(conn.fetch)\n' }, []], + multiline_tuple_local: [{ 'm.py': header + 'def cb(pool):\n (a,\n conn) = 1, Decoy()\n pool.submit(conn.fetch)\n' }, []], + multiline_tuple_rebind: [{ 'm.py': header + 'def reset():\n global conn\n (a,\n conn) = 1, Decoy()\n\ndef cb(pool):\n pool.submit(conn.fetch)\n' }, []], + annotation_contradicts_value: [{ 'm.py': 'from store import Store\nfrom decoy import Decoy\nconn: Store = Decoy()\n\n' + + 'def cb(pool):\n pool.submit(conn.fetch)\n' }, []], + annotation_over_factory: [{ 'm.py': 'from store import Store\nconn: Store = make_store()\n\ndef cb(pool):\n pool.submit(conn.fetch)\n' }, + ['cb -> Store::fetch']], + external_production_write: [{ 'settings.py': header, + 'reset.py': 'import settings\nfrom decoy import Decoy\n\ndef reset():\n settings.conn = Decoy()\n', + 'c.py': 'import settings\n\ndef cb(pool):\n pool.submit(settings.conn.fetch)\n' }, []], + external_aliased_write: [{ 'pkg/__init__.py': '', 'pkg/settings.py': header, + 'reset.py': 'import pkg.settings as cfg\n\ndef reset():\n cfg.conn = make_conn()\n', + 'c.py': 'from pkg import settings\n\ndef cb(pool):\n pool.submit(settings.conn.fetch)\n' }, []], + external_relative_write: [{ 'pkg/__init__.py': '', 'pkg/settings.py': header, + 'pkg/reset.py': 'from . import settings\nfrom decoy import Decoy\n\ndef reset():\n settings.conn = Decoy()\n', + 'c.py': 'from pkg import settings\n\ndef cb(pool):\n pool.submit(settings.conn.fetch)\n' }, []], + external_setattr: [{ 'settings.py': header, + 'reset.py': 'import settings\nfrom decoy import Decoy\n\ndef reset():\n setattr(settings, "conn", Decoy())\n', + 'c.py': 'import settings\n\ndef cb(pool):\n pool.submit(settings.conn.fetch)\n' }, []], + external_subclass_write: [{ 'settings.py': header, + 'sub.py': 'from store import Store\nclass SubStore(Store):\n def fetch(self, ids):\n return ids\n', + 'reset.py': 'import settings\nfrom sub import SubStore\n\ndef reset():\n settings.conn = SubStore()\n', + 'c.py': 'import settings\n\ndef cb(pool):\n pool.submit(settings.conn.fetch)\n' }, ['cb -> Store::fetch']], + test_double_write: [{ 'settings.py': header, + 'tests/test_c.py': 'import settings\nfrom unittest.mock import MagicMock\n\ndef test_cb(pool):\n settings.conn = MagicMock()\n pool.submit(settings.conn.fetch)\n', + 'c.py': 'import settings\n\ndef cb(pool):\n pool.submit(settings.conn.fetch)\n' }, ['cb -> Store::fetch']], + test_monkeypatch: [{ 'settings.py': header, + 'tests/test_c.py': 'import settings\n\ndef test_cb(monkeypatch, pool):\n monkeypatch.setattr(settings, "conn", object())\n pool.submit(settings.conn.fetch)\n' }, []], + globals_literal_write: [{ 'm.py': header + 'def reset():\n globals()["conn"] = Decoy()\n\ndef cb(pool):\n pool.submit(conn.fetch)\n' }, []], + globals_dynamic_write: [{ 'm.py': header + 'for name, obj in [("conn", Decoy())]:\n globals()[name] = obj\n\ndef cb(pool):\n pool.submit(conn.fetch)\n' }, []], + unrelated_attribute_write: [{ 'settings.py': header, + 'other.py': 'from decoy import Decoy\n\nclass Holder:\n def __init__(self):\n self.conn = Decoy()\n', + 'c.py': 'import settings\n\ndef cb(pool):\n pool.submit(settings.conn.fetch)\n' }, ['cb -> Store::fetch']], + // A write lands on the module its import names, never on another module sharing its tail. + write_path_is_exact: [{ 'x/__init__.py': '', 'y/__init__.py': '', 'x/settings.py': header, 'y/settings.py': header, + 'reset.py': 'from x import settings\nfrom decoy import Decoy\n\ndef reset():\n settings.conn = Decoy()\n', + 'cx.py': 'from x import settings\n\ndef cb_x(pool):\n pool.submit(settings.conn.fetch)\n', + 'cy.py': 'from y import settings\n\ndef cb_y(pool):\n pool.submit(settings.conn.fetch)\n' }, ['cb_y -> Store::fetch']], + ambiguous_write_target: [{ 'x/__init__.py': '', 'y/__init__.py': '', 'x/settings.py': header, 'y/settings.py': header, + 'reset.py': 'import settings\nfrom decoy import Decoy\n\ndef reset():\n settings.conn = Decoy()\n', + 'cx.py': 'from x import settings\n\ndef cb_x(pool):\n pool.submit(settings.conn.fetch)\n', + 'cy.py': 'from y import settings\n\ndef cb_y(pool):\n pool.submit(settings.conn.fetch)\n' }, []], + relative_write_is_exact: [{ 'pkg/__init__.py': '', 'pkg/sub/__init__.py': '', 'pkg/settings.py': header, + 'other/__init__.py': '', 'other/pkg/__init__.py': '', 'other/pkg/settings.py': header, + 'pkg/sub/reset.py': 'from .. import settings\n\ndef reset():\n settings.conn = make_conn()\n', + 'c2.py': 'from other.pkg import settings\n\ndef cb2(pool):\n pool.submit(settings.conn.fetch)\n' }, ['cb2 -> Store::fetch']], + namespace_import_binds_root: [{ 'pkg/__init__.py': '', 'pkg/settings.py': header, + 'c.py': 'import pkg.settings\nfrom decoy import Decoy\n\nsettings = Bag()\nsettings.conn = Decoy()\n', + 'd.py': 'from pkg import settings\n\ndef cb(pool):\n pool.submit(settings.conn.fetch)\n' }, ['cb -> Store::fetch']], + // `other.pkg.settings as settings` is not an explicit alias of `pkg.settings`: the + // write goes to the other module, so pkg's global keeps its type (the consumer's + // relative import names pkg/settings.py exactly). + alias_of_longer_module_is_not_explicit: [{ 'pkg/__init__.py': '', 'pkg/settings.py': header, + 'other/__init__.py': '', 'other/pkg/__init__.py': '', 'other/pkg/settings.py': 'conn = None\n', + 'reset.py': 'import pkg.settings\nimport other.pkg.settings as settings\nfrom decoy import Decoy\n\ndef reset():\n settings.conn = Decoy()\n', + 'pkg/d.py': 'from . import settings\n\ndef cb(pool):\n pool.submit(settings.conn.fetch)\n' }, ['cb -> Store::fetch']], + alias_in_string_is_not_explicit: [{ 'pkg/__init__.py': '', 'pkg/settings.py': 'conn = None\n', + 'reset.py': 'import pkg.settings\nfrom decoy import Decoy\nnote = "import pkg.settings as settings"\n\ndef reset():\n settings.conn = Decoy()\n', + 'd.py': 'from pkg import settings\n\ndef cb(pool):\n pool.submit(settings.conn.fetch)\n' }, []], + explicit_alias_equal_to_leaf: [{ 'pkg/__init__.py': '', 'pkg/settings.py': header, + 'reset.py': 'import pkg.settings as settings\nfrom decoy import Decoy\n\ndef reset():\n settings.conn = Decoy()\n', + 'd.py': 'from pkg import settings\n\ndef cb(pool):\n pool.submit(settings.conn.fetch)\n' }, []], + one_line_main_write: [{ 'settings.py': header, + 'run.py': 'import settings\nfrom decoy import Decoy\n\nif __name__ == "__main__": settings.conn = Decoy()\n', + 'c.py': 'import settings\n\ndef cb(pool):\n pool.submit(settings.conn.fetch)\n' }, ['cb -> Store::fetch']], + globals_in_main_block: [{ 'm.py': header + 'if __name__ == "__main__":\n globals()["conn"] = Decoy()\n\ndef cb(pool):\n pool.submit(conn.fetch)\n' }, + ['cb -> Store::fetch']], + vars_in_function_is_locals: [{ 'm.py': header + 'def dump():\n return vars()\n\ndef cb(pool):\n pool.submit(conn.fetch)\n' }, + ['cb -> Store::fetch']], + example_write_is_production: [{ 'settings.py': header, + 'examples/wire.py': 'import settings\nfrom decoy import Decoy\n\ndef wire():\n settings.conn = Decoy()\n', + 'c.py': 'import settings\n\ndef cb(pool):\n pool.submit(settings.conn.fetch)\n' }, []], + conftest_write_is_test: [{ 'settings.py': header, + 'pkg/conftest.py': 'import settings\nfrom decoy import Decoy\n\ndef fixture():\n settings.conn = Decoy()\n', + 'c.py': 'import settings\n\ndef cb(pool):\n pool.submit(settings.conn.fetch)\n' }, ['cb -> Store::fetch']], + main_block_write: [{ 'settings.py': header, + 'run.py': 'import settings\nfrom decoy import Decoy\n\nif __name__ == "__main__":\n settings.conn = Decoy()\n', + 'c.py': 'import settings\n\ndef cb(pool):\n pool.submit(settings.conn.fetch)\n' }, ['cb -> Store::fetch']], + own_main_block_write: [{ 'm.py': header + 'if __name__ == "__main__":\n conn = Decoy()\n\ndef cb(pool):\n pool.submit(conn.fetch)\n' }, + ['cb -> Store::fetch']], + globals_continued_key: [{ 'm.py': header + 'def reset():\n globals()[\n "conn"] = Decoy()\n\ndef cb(pool):\n pool.submit(conn.fetch)\n' }, []], + globals_alias: [{ 'm.py': header + 'def reset():\n g = globals()\n g["conn"] = Decoy()\n\ndef cb(pool):\n pool.submit(conn.fetch)\n' }, []], + globals_in_string: [{ 'm.py': header + 'NOTE = "globals().update(mapping)"\n\ndef cb(pool):\n pool.submit(conn.fetch)\n' }, + ['cb -> Store::fetch']], + external_dict_write: [{ 'settings.py': header, + 'reset.py': 'import settings\nfrom decoy import Decoy\n\ndef reset():\n settings.__dict__["conn"] = Decoy()\n', + 'c.py': 'import settings\n\ndef cb(pool):\n pool.submit(settings.conn.fetch)\n' }, []], + external_annotated_write: [{ 'settings.py': header, + 'reset.py': 'import settings\nfrom store import Store\n\ndef reset():\n settings.conn: Store = Store()\n', + 'c.py': 'import settings\n\ndef cb(pool):\n pool.submit(settings.conn.fetch)\n' }, ['cb -> Store::fetch']], + external_parenthesized_write: [{ 'settings.py': header, + 'reset.py': 'import settings\nfrom store import Store\n\ndef reset():\n settings.conn = (Store())\n', + 'c.py': 'import settings\n\ndef cb(pool):\n pool.submit(settings.conn.fetch)\n' }, ['cb -> Store::fetch']], + dotted_import_collision: [{ 'alpha/__init__.py': '', 'beta/__init__.py': '', + 'alpha/foo.py': 'class Client:\n def fetch(self):\n return 1\n', 'beta/foo.py': 'class Client:\n def fetch(self):\n return 2\n', + 'm.py': 'import beta.foo\nimport alpha.foo\nconn = None\n\ndef init():\n global conn\n conn = alpha.foo.Client()\n\ndef cb(pool):\n pool.submit(conn.fetch)\n' }, []], + dotted_base_collision: [{ 'alpha/__init__.py': '', 'beta/__init__.py': '', + 'alpha/foo.py': 'class Client:\n def fetch(self):\n return 1\n', 'beta/foo.py': 'class Client:\n def fetch(self):\n return 2\n', + 'm.py': 'import beta.foo\nimport alpha.foo\n\nclass Sub(alpha.foo.Client):\n pass\n\ndef go(pool, s: Sub):\n pool.submit(s.fetch)\n' }, []], + }; + const got: Record = {}; + for (const [name, [files, _]] of Object.entries(cases)) { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), `cg-fnref-global-${name}-`)); + const all = { 'store.py': 'class Store:\n def fetch(self, ids):\n return ids\n', + 'decoy.py': 'class Decoy:\n def fetch(self, ids):\n return []\n', ...files }; + for (const [file, content] of Object.entries(all)) { + fs.mkdirSync(path.dirname(path.join(tmpDir, file)), { recursive: true }); + fs.writeFileSync(path.join(tmpDir, file), content); + } + const cg = CodeGraph.initSync(tmpDir); + try { + await cg.indexAll(); + got[name] = cg.getNodesByName('fetch').flatMap(t => cg.getIncomingEdges(t.id) + .filter(e => e.kind === 'references' && e.metadata?.fnRef === true) + .map(e => `${cg.getNode(e.source)?.name} -> ${t.qualifiedName}`)).sort(); + } finally { cg.close(); fs.rmSync(tmpDir, { recursive: true, force: true }); tmpDir = undefined; } + } + expect(got).toEqual(Object.fromEntries(Object.entries(cases).map(([name, [, want]]) => [name, want]))); + }, 60_000); + + it('#1820: a module global re-resolves after its module changes (sync)', async () => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-fnref-global-sync-')); + const settings = 'from store import Store\nfrom decoy import Decoy\nconn = None\n\ndef init():\n global conn\n conn = Store()\n'; + const consumer = 'import settings\n\ndef cb(pool):\n pool.submit(settings.conn.fetch)\n'; + fs.writeFileSync(path.join(tmpDir, 'store.py'), 'class Store:\n def fetch(self, ids):\n return ids\n'); + fs.writeFileSync(path.join(tmpDir, 'decoy.py'), 'class Decoy:\n def fetch(self, ids):\n return []\n'); + fs.writeFileSync(path.join(tmpDir, 'settings.py'), settings); + fs.writeFileSync(path.join(tmpDir, 'consumer.py'), consumer); + const cg = CodeGraph.initSync(tmpDir); + const edges = () => cg.getNodesByName('fetch').flatMap(t => cg.getIncomingEdges(t.id) + .filter(e => e.metadata?.fnRef === true).map(e => `${cg.getNode(e.source)?.name} -> ${t.qualifiedName}`)); + try { + await cg.indexAll(); + expect(edges()).toEqual(['cb -> Store::fetch']); + fs.writeFileSync(path.join(tmpDir, 'settings.py'), settings.replace('conn = Store()', 'conn = Decoy()')); + fs.writeFileSync(path.join(tmpDir, 'consumer.py'), consumer + '\n# touched\n'); + await cg.sync(); + expect(edges()).toEqual(['cb -> Decoy::fetch']); + } finally { cg.close(); } + }); + it('#1820: Go receiver types disambiguate method values and reject external fields', async () => { tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-fnref-go-scope-')); fs.writeFileSync(path.join(tmpDir, 'main.go'), `package demo diff --git a/__tests__/resolution.test.ts b/__tests__/resolution.test.ts index 9bdf768cd..14b5198f4 100644 --- a/__tests__/resolution.test.ts +++ b/__tests__/resolution.test.ts @@ -648,6 +648,28 @@ from ..services import auth_service expect(mappings.some((m) => m.localName === 'helper')).toBe(true); expect(mappings.some((m) => m.localName === 'User')).toBe(true); }); + + it('should extract parenthesized Python from-imports', () => { + const content = ` +from store.base import ( + Base, + helper as h, # a comment, with a comma and a call() +) +""" +from docstring import NotAnImport +""" +from store.rows import (Row, Col) +`; + + const mappings = extractImportMappings('src/main.py', content, 'python'); + + expect(mappings.map((m) => [m.localName, m.exportedName, m.source])).toEqual([ + ['Base', 'Base', 'store.base'], + ['h', 'helper', 'store.base'], + ['Row', 'Row', 'store.rows'], + ['Col', 'Col', 'store.rows'], + ]); + }); }); describe('JVM FQN Import Resolution', () => { diff --git a/src/resolution/import-resolver.ts b/src/resolution/import-resolver.ts index dc3b58395..7fa9912e2 100644 --- a/src/resolution/import-resolver.ts +++ b/src/resolution/import-resolver.ts @@ -11,6 +11,7 @@ import { UnresolvedRef, ResolvedRef, ResolutionContext, ImportMapping, ReExport import { applyAliases } from './path-aliases'; import { extractLocalExportAliases } from './alias-binding'; import { resolveWorkspaceImport } from './workspace-packages'; +import { stripCommentsForRegex } from './strip-comments'; import { resolveMethodOnType, resolveObjectLiteralMember, @@ -937,7 +938,7 @@ export function extractImportMappings( // whole SFC (markup + styles included) is safe. mappings.push(...extractJSImports(content)); } else if (language === 'python') { - mappings.push(...extractPythonImports(content)); + mappings.push(...extractPythonImports(stripCommentsForRegex(content, 'python'))); } else if (language === 'go') { mappings.push(...extractGoImports(content)); } else if (language === 'java' || language === 'kotlin') { @@ -1066,13 +1067,14 @@ function extractJSImports(content: string): ImportMapping[] { function extractPythonImports(content: string): ImportMapping[] { const mappings: ImportMapping[] = []; - // from X import Y - const fromImportRegex = /from\s+([\w.]+)\s+import\s+([^#\n]+)/g; + // from X import Y, and the parenthesized form `from X import (\n Y,\n Z,\n)` + const fromImportRegex = /from\s+([\w.]+)\s+import\s+(\([^)]*\)|[^#\n]+)/g; let match; while ((match = fromImportRegex.exec(content)) !== null) { const [, source, imports] = match; - const names = imports!.split(',').map((s) => s.trim()); + const names = imports!.trim().replace(/^\(|\)$/g, '') + .split(',').map((s) => s.trim()); for (const name of names) { const aliasMatch = name.match(/(\w+)\s+as\s+(\w+)/); diff --git a/src/resolution/name-matcher.ts b/src/resolution/name-matcher.ts index 1f5d6f8a2..701c0ea03 100644 --- a/src/resolution/name-matcher.ts +++ b/src/resolution/name-matcher.ts @@ -281,6 +281,17 @@ function matchMemberFunctionRef(ref: UnresolvedRef, context: ResolutionContext): if (ref.language === 'python') { const cls = pythonRefClass(receiver, ref, context); if (cls) return result(pythonMembers(cls, member, ref, context)); + // `mod.global` / an imported `global` itself — never a deeper chain (`mod.global.field`). + const segments = receiver.split('.'); + const hit = segments.length <= 2 ? context.resolveImport?.({ ...ref, referenceName: receiver, referenceKind: 'references' }) : null; + const global = hit && context.getNodeById?.(hit.targetNodeId); + // A local import of the root binds what the file imports, unless the file imports it from two places. + if (global && global.name === segments[segments.length - 1] && isPythonModuleGlobal(global, context) && + pythonImportKeys(segments[0]!, ref.filePath, context).size === 1 && + !pythonGlobalBindings(segments[0]!, ref.filePath, context).some(b => b.kind !== 'import') && + !pythonBindsLocally(segments[0]!, ref, context, false)) { + return result(pythonGlobalMembers(global, member, ref, context)); + } } const imported = context.resolveImport?.(ref); const node = imported && context.getNodeById?.(imported.targetNodeId); @@ -305,6 +316,11 @@ function matchMemberFunctionRef(ref: UnresolvedRef, context: ResolutionContext): type = pythonFieldType(receiver, owner, ref, context); } else { type = pythonLocalType(receiver, ref, context); + const global = type === null + ? context.getNodesInFile(ref.filePath).find(n => n.name === receiver && isPythonModuleGlobal(n, context)) : undefined; + if (global && !pythonBindsLocally(receiver, ref, context, true)) { + return result(pythonGlobalMembers(global, member, ref, context)); + } } // A type name used directly (`Store.fetch`) is scoped just like an annotation. if (!type && /^[A-Z]\w*$/.test(receiver)) type = receiver; @@ -337,6 +353,14 @@ function matchMemberFunctionRef(ref: UnresolvedRef, context: ResolutionContext): function pythonRefClass(name: string, ref: UnresolvedRef, context: ResolutionContext): Node | null { const imports = context.getImportMappings(ref.filePath, 'python'); + // `import pkg.mod` then `pkg.mod.Cls`: the mapping keys the module by its last segment. + const module = imports.find(i => i.isNamespace && name.startsWith(`${i.source}.`) && + /^\w+$/.test(name.slice(i.source.length + 1))); + if (module) { + // Two modules with the same last segment (`import a.foo`, `import b.foo`) share the key: refuse. + if (imports.filter(i => i.localName === module.localName).length !== 1) return null; + name = `${module.localName}.${name.slice(module.source.length + 1)}`; + } if (imports.some(i => i.localName === name.split('.')[0])) { const hit = context.resolveImport?.({ ...ref, referenceName: name, referenceKind: 'references' }); const node = hit && context.getNodeById?.(hit.targetNodeId); @@ -411,6 +435,478 @@ function pythonLocalType(receiver: string, ref: UnresolvedRef, context: Resoluti return caller?.signature?.match(new RegExp(`\\b${receiver}\\s*:\\s*["']?([\\w.]+)`))?.[1] ?? null; } +/** A module-scope Python variable (not a class attribute or a function local). */ +function isPythonModuleGlobal(node: Node, context: ResolutionContext): boolean { + return node.language === 'python' && (node.kind === 'variable' || node.kind === 'constant') && + !context.getNodesInFile(node.filePath).some(n => + (n.kind === 'class' || n.kind === 'function' || n.kind === 'method') && + n.startLine <= node.startLine && n.endLine >= node.startLine); +} + +/** How one line binds a name: `global`, a plain `name = value` / `name: T`, an import, or any other binding. */ +type PythonBinding = + | { kind: 'global' } + | { kind: 'assign'; type: string | null; value: string; line: number } + | { kind: 'import'; key: string } + | { kind: 'other' }; + +const PYTHON_STATEMENT_STARTS = new WeakMap>(); +/** Per line: does it start a statement (bracket depth 0, no `\` continuation)? String contents are skipped. */ +function pythonStatementStarts(filePath: string, context: ResolutionContext): boolean[] { + let files = PYTHON_STATEMENT_STARTS.get(context); + if (!files) { files = new Map(); PYTHON_STATEMENT_STARTS.set(context, files); } + let starts = files.get(filePath); + if (starts) return starts; + starts = []; + let depth = 0; + let continued = false; + for (const line of pythonMemberLines(filePath, context)) { + starts.push(depth === 0 && !continued); + let quote = ''; + for (let i = 0; i < line.length; i++) { + const c = line[i]!; + if (quote) { if (c === '\\') i++; else if (c === quote) quote = ''; continue; } + if (c === '"' || c === "'") quote = c; + else if (c === '(' || c === '[' || c === '{') depth++; + else if (c === ')' || c === ']' || c === '}') depth = Math.max(0, depth - 1); + } + continued = /\\\s*$/.test(line); + } + files.set(filePath, starts); + return starts; +} + +/** Line indexes that belong to `scope` itself (null: the module), not to a def or class nested in it. */ +function pythonOwnLines(scope: Node | null, filePath: string, context: ResolutionContext): number[] { + const count = pythonMemberLines(filePath, context).length; + const from = scope ? scope.startLine : 1; + const to = scope ? Math.min(scope.endLine, count) : count; + const nested = new Uint8Array(to - from + 1); + for (const n of context.getNodesInFile(filePath)) { + if ((n.kind !== 'function' && n.kind !== 'method' && n.kind !== 'class') || n.id === scope?.id) continue; + if (n.startLine < from || n.endLine > to || (scope && n.startLine <= scope.startLine)) continue; + nested.fill(1, n.startLine - from, n.endLine - from + 1); + } + const own: number[] = []; + for (let l = from; l <= to; l++) if (!nested[l - from]) own.push(l - 1); + return own; +} + +/** Split `a = b = value` at its top-level assignment operators; null when the line assigns nothing. */ +function pythonAssignment(line: string): { targets: string[]; value: string; augmented: boolean } | null { + const targets: string[] = []; + let depth = 0; + let quote = ''; + let start = 0; + let augmented = false; + for (let i = 0; i < line.length; i++) { + const c = line[i]!; + if (quote) { if (c === '\\') i++; else if (c === quote) quote = ''; continue; } + if (c === '"' || c === "'") { quote = c; continue; } + if (c === '(' || c === '[' || c === '{') { depth++; continue; } + if (c === ')' || c === ']' || c === '}') { depth--; continue; } + if (c !== '=' || depth !== 0) continue; + const prev = line[i - 1] ?? ''; + if (line[i + 1] === '=') { i++; continue; } // == + if (prev === '!' || prev === ':') continue; // != and the walrus + if ((prev === '<' || prev === '>') && line[i - 2] !== prev) continue; // <= >= + const op = line.slice(start, i).match(/(?:\/\/|\*\*|>>|<<|[-+*/%&|^@])$/)?.[0]; + if (op) augmented = true; + targets.push(line.slice(start, i - (op?.length ?? 0))); + start = i + 1; + } + return targets.length ? { targets, value: line.slice(start), augmented } : null; +} + +/** + * Every way the statement starting at line `index` binds `name`, read across + * its continuation lines. Only statement lines can assign or import; any line + * can bind through `as`, a loop, a lambda or the walrus. + */ +function pythonLineBindings(lines: string[], index: number, statements: boolean[], name: string): PythonBinding[] { + let line = lines[index]!; + if (statements[index]) for (let j = index + 1; j < lines.length && !statements[j]; j++) line += '\n' + lines[j]; + const out: PythonBinding[] = []; + // An import statement can name `name` on a continuation line: `from x import (\n name,\n)`. + const imported = statements[index] ? line.match(/^\s*(?:from\s+([\w.]+)\s+)?import\s+([\s\S]*)$/) : null; + if (imported) { + const names = imported[2]!; + // `from x import *` can bind any name. + if (imported[1] && names.trim() === '*') return [{ kind: 'other' }]; + if (!names.includes(name)) return out; + for (const part of names.replace(/[()\\]/g, ' ').split(',')) { + const m = part.trim().match(/^([\w.]+)(?:\s+as\s+(\w+))?$/); + const local = m && (m[2] ?? (imported[1] ? m[1]! : m[1]!.split('.')[0]!)); + if (local !== name) continue; + out.push({ kind: 'import', key: imported[1] ? `${imported[1]}:${m![1]}` : `${m![2] ? m![1] : local}:*` }); + } + return out; + } + if (!line.includes(name)) return out; + const word = new RegExp(`(? s.trim() === name) + ? [declared[1] === 'global' ? { kind: 'global' } : { kind: 'other' }] : []; + } + if (statements[index]) { + // A `case` pattern binds its capture names (`case [name]:`, `case Cls(k=name):`, `case name:`). + if (/^\s*case\b/.test(line)) return [{ kind: 'other' }]; + const assignment = pythonAssignment(line); + if (assignment) { + const target = assignment.targets.length === 1 && !assignment.augmented ? assignment.targets[0]!.trim() : ''; + const annotated = target.match(new RegExp(`^${name}\\s*:\\s*["']?([\\w.]+)["']?$`)); + if (target === name || annotated) { + out.push({ kind: 'assign', type: annotated?.[1] ?? null, value: assignment.value.trim(), line: index }); + } else if (assignment.targets.some(t => word.test(t))) { + out.push({ kind: 'other' }); + } + } else { + const annotated = line.match(new RegExp(`^\\s*${name}\\s*:\\s*["']?([\\w.]+)["']?\\s*$`)); + if (annotated) out.push({ kind: 'assign', type: annotated[1]!, value: '', line: index }); + else if (new RegExp(`^\\s*del\\b`).test(line)) out.push({ kind: 'other' }); + } + } + if (new RegExp(`\\b${name}[ \\t]*:=|\\bas[ \\t]+${name}\\b`).test(line)) out.push({ kind: 'other' }); + for (const loop of line.matchAll(/\bfor\s+([^:]+?)\s+in\b/g)) if (word.test(loop[1]!)) out.push({ kind: 'other' }); + for (const lambda of line.matchAll(/\blambda\b([^:]*):/g)) if (word.test(lambda[1]!)) out.push({ kind: 'other' }); + return out; +} + +/** + * Whether `name`, read at the ref, is bound by the calling function or one that + * encloses it (parameter, assignment, loop, `as`, lambda, import) rather than + * being the module global. With `importsBind` false, an import of the name is + * not a shadow: it binds the same module the file imports. + */ +function pythonBindsLocally(name: string, ref: UnresolvedRef, context: ResolutionContext, importsBind: boolean): boolean { + const lines = pythonMemberLines(ref.filePath, context); + const statements = pythonStatementStarts(ref.filePath, context); + const param = new RegExp(`[(,]\\s*\\*{0,2}${name}\\s*[:=,)]`); + const scopes = context.getNodesInFile(ref.filePath).filter(n => + (n.kind === 'function' || n.kind === 'method') && n.startLine <= ref.line && n.endLine >= ref.line) + .sort((a, b) => b.startLine - a.startLine); + for (const scope of scopes) { + const own = pythonOwnLines(scope, ref.filePath, context); + const bindings = own.flatMap(i => pythonLineBindings(lines, i, statements, name)); + if (bindings.some(b => b.kind === 'global')) return false; + const def = own.find(i => /^\s*(?:async\s+)?def\b/.test(lines[i]!)); + let header = scope.signature ?? ''; + for (let j = def ?? lines.length; j < lines.length && (j === def || !statements[j]); j++) header += lines[j]; + if (param.test(header)) return true; + if (bindings.some(b => b.kind !== 'global' && (b.kind !== 'import' || importsBind))) return true; + } + return false; +} + +/** Distinct sources a file imports `name` from, at any scope (`from a import x` → `a:x`). */ +function pythonImportKeys(name: string, filePath: string, context: ResolutionContext): Set { + return pythonNameScan(context, `imports\0${filePath}\0${name}`, () => scanPythonImportKeys(name, filePath, context)); +} + +const PYTHON_NAME_SCANS = new WeakMap>(); +/** Per-file, per-name scans are shared by every ref in the file; cleared with the other memos on sync. */ +function pythonNameScan(context: ResolutionContext, key: string, scan: () => T): T { + let memo = PYTHON_NAME_SCANS.get(context); + if (!memo) { memo = new Map(); PYTHON_NAME_SCANS.set(context, memo); } + if (!memo.has(key)) memo.set(key, scan()); + return memo.get(key) as T; +} + +function scanPythonImportKeys(name: string, filePath: string, context: ResolutionContext): Set { + const lines = pythonMemberLines(filePath, context); + const statements = pythonStatementStarts(filePath, context); + const keys = new Set(); + for (let i = 0; i < lines.length; i++) { + if (!statements[i] || !/\bimport\b/.test(lines[i]!)) continue; + for (const b of pythonLineBindings(lines, i, statements, name)) if (b.kind === 'import') keys.add(b.key); + } + return keys; +} + +/** Every binding of module global `name` in `filePath`: at module scope, and in each function that declares it `global`. */ +function pythonGlobalBindings(name: string, filePath: string, context: ResolutionContext): PythonBinding[] { + return pythonNameScan(context, `globals\0${filePath}\0${name}`, () => scanPythonGlobalBindings(name, filePath, context)); +} + +function scanPythonGlobalBindings(name: string, filePath: string, context: ResolutionContext): PythonBinding[] { + const lines = pythonMemberLines(filePath, context); + const statements = pythonStatementStarts(filePath, context); + const scopes = context.getNodesInFile(filePath).filter(n => n.kind === 'class' || n.kind === 'function' || n.kind === 'method'); + const regions = [pythonOwnLines(null, filePath, context)]; + for (let i = 0; i < lines.length; i++) { + if (!/^\s*global\b/.test(lines[i]!) || !pythonLineBindings(lines, i, statements, name).length) continue; + const scope = scopes.filter(n => n.startLine <= i + 1 && n.endLine >= i + 1).sort((a, b) => b.startLine - a.startLine)[0]; + if (scope && scope.kind !== 'class') regions.push(pythonOwnLines(scope, filePath, context)); + } + const script = pythonMainBlockLines(filePath, context); + return regions.flatMap(region => region.filter(i => !script.has(i)).flatMap(i => pythonLineBindings(lines, i, statements, name))) + .filter(b => b.kind !== 'global'); +} + +const PYTHON_GLOBAL_CLASSES = new WeakMap>(); +/** + * The classes a module global can hold. It has no static type (`conn = None`, + * rebound by `global conn; conn = Backend()`), so its type is the set of + * classes its module assigns to it: at module scope, or in a function that + * declares it `global`. Any other binding of it there (an opaque value, a + * tuple target, `for`/`with`/import, a star import, a `globals()` write) + * makes the type unknown (null). Production writes from other modules + * (`settings.conn = X`) join the set; see pythonExternalWrites. + */ +function pythonGlobalClasses(global: Node, ref: UnresolvedRef, context: ResolutionContext): Node[] | null { + let memo = PYTHON_GLOBAL_CLASSES.get(context); + if (!memo) { memo = new Map(); PYTHON_GLOBAL_CLASSES.set(context, memo); } + if (memo.has(global.id)) return memo.get(global.id)!; + const file = global.filePath; + const external = pythonExternalWrites(global, context); + // Each type is resolved in the file that wrote it (its imports name the class). + const writes: Array<{ type: string; file: string }> = [...external.writes]; + let classes: Node[] | null = external.unknown || pythonDynamicGlobalWrite(global.name, file, context) ? null : []; + for (const b of classes ? pythonGlobalBindings(global.name, file, context) : []) { + if (b.kind !== 'assign') { classes = null; break; } + const constructor = b.value && b.value !== 'None' ? pythonConstructorCall(b.value) : null; + if (b.type) { + // `conn: Base = make()` trusts the annotation; `conn: A = B()` contradicts it. + if (constructor && constructor.split('.').pop() !== b.type.split('.').pop()) { classes = null; break; } + writes.push({ type: b.type, file }); + continue; + } + if (b.value === 'None') continue; + if (!constructor) { classes = null; break; } + writes.push({ type: constructor, file }); + } + const seen = new Set(); + for (const write of classes ? writes : []) { + const cls = pythonRefClass(write.type, { ...ref, filePath: write.file }, context); + if (!cls) { classes = null; break; } + if (!seen.has(cls.id)) { seen.add(cls.id); classes!.push(cls); } + } + memo.set(global.id, classes); + return classes; +} + +/** + * The repo files a Python module path can name from `fromFile`. Relative + * paths (`..settings`) resolve exactly; absolute ones match a file path + * suffix, so a source root (`src/`) still resolves — and two files sharing + * the tail (`x/settings.py`, `y/settings.py`) both come back. + */ +function pythonModuleFiles(dotted: string, fromFile: string, context: ResolutionContext): string[] { + const dots = dotted.match(/^\.+/)?.[0].length ?? 0; + const parts = dotted.slice(dots).split('.').filter(Boolean); + if (!parts.length) return []; + let dir = ''; + if (dots) { + dir = path.posix.dirname(fromFile.replace(/\\/g, '/')); + for (let i = 1; i < dots; i++) dir = path.posix.dirname(dir); + if (dir === '.') dir = ''; + } + const rel = [dir, ...parts].filter(Boolean).join('/'); + const matches = (file: string, want: string) => dots ? file === want : file === want || file.endsWith(`/${want}`); + const last = parts[parts.length - 1]!; + return [ + ...context.getNodesByName(`${last}.py`), ...context.getNodesByName(`${last}.pyi`), ...context.getNodesByName('__init__.py'), + ].filter(n => n.kind === 'file' && (matches(n.filePath, `${rel}.py`) || matches(n.filePath, `${rel}.pyi`) || matches(n.filePath, `${rel}/__init__.py`))) + .map(n => n.filePath); +} + +/** + * How `filePath` spells the module `moduleFile`: `aliases` import exactly that + * file; `ambiguous` could also be another file sharing its dotted tail. + * `import a.b` binds `a`, so that module is spelled `a.b` (the mapping's + * last-segment `localName` is not a binding); `import a.b as c` binds `c`. + */ +function pythonModuleAliases(filePath: string, moduleFile: string, context: ResolutionContext): { aliases: string[]; ambiguous: string[] } { + const aliases = new Set(); + const ambiguous = new Set(); + for (const m of context.getImportMappings(filePath, 'python')) { + const dotted = m.isNamespace ? m.source + : /^\.+$/.test(m.source) ? `${m.source}${m.exportedName}` : `${m.source}.${m.exportedName}`; + const files = pythonModuleFiles(dotted, filePath, context); + if (!files.includes(moduleFile)) continue; + // The mapping cannot tell `import a.b` from `import a.b as b`; the source line can. + // Exactly this module (not `other.a.b`), outside string literals. + const explicit = new RegExp(`\\bimport\\s[^\\n]*(? explicit.test(blankPythonStrings(line))); + (files.length === 1 ? aliases : ambiguous).add(plainDotted ? m.source : m.localName); + } + return { aliases: [...aliases], ambiguous: [...ambiguous] }; +} + +const PYTHON_MAIN_GUARD = /^if\s+(?:__name__\s*==\s*(['"])__main__\1|(['"])__main__\2\s*==\s*__name__)\s*:/; +/** Line indexes inside a top-level `if __name__ == "__main__":` block — script code, not module state. */ +function pythonMainBlockLines(filePath: string, context: ResolutionContext): Set { + return pythonNameScan(context, `main\0${filePath}`, () => { + const lines = pythonMemberLines(filePath, context); + const inside = new Set(); + for (let i = 0; i < lines.length; i++) { + const guard = lines[i]!.match(PYTHON_MAIN_GUARD); + if (!guard) continue; + if (lines[i]!.slice(guard[0].length).trim()) inside.add(i); // `if __name__ == "__main__": stmt` + for (let j = i + 1; j < lines.length && !/^\S/.test(lines[j]!); j++) inside.add(j); + } + return inside; + }); +} + +/** Blank single-line string contents (triple-quoted strings are already blanked by the comment stripper). */ +function blankPythonStrings(text: string): string { + return text.replace(/(['"])(?:\\.|(?!\1)[^\\\n])*\1/g, m => m[0] + ' '.repeat(m.length - 2) + m[0]); +} + +/** Test code installs doubles: the narrow test-suite set, plus pytest's `conftest.py` wherever it sits. */ +function isPythonTestFile(filePath: string): boolean { + return isTestPath(filePath) || /(?:^|\/)conftest\.py$/.test(filePath.replace(/\\/g, '/')); +} + +/** + * Writes to module global `global` from OTHER files. A production write + * `. = Cls(...)` adds a type; any other production write + * (another value, a tuple target, `setattr`) makes the type unknown. Test + * files install doubles (`settings.conn = MagicMock()`, `monkeypatch.setattr`) + * that do not define the production type; they are recorded as `writers`, and + * a ref inside a writer resolves nothing through the global. + */ +function pythonExternalWrites(global: Node, context: ResolutionContext): { writes: Array<{ type: string; file: string }>; unknown: boolean; writers: Set } { + return pythonNameScan(context, `external\0${global.id}`, () => { + const out = { writes: [] as Array<{ type: string; file: string }>, unknown: false, writers: new Set() }; + const name = global.name; + const escape = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + for (const file of context.getAllFiles()) { + if (file === global.filePath || !/\.pyi?$/.test(file) || !context.readFile(file)?.includes(name)) continue; + const test = isPythonTestFile(file); + const { aliases, ambiguous } = pythonModuleAliases(file, global.filePath, context); + if (!test && !aliases.length && !ambiguous.length) continue; + const spell = (names: string[]) => `(?:${names.map(escape).join('|')})`; + // Test files: any receiver (the double may sit on any spelling of the module). + const receiver = test ? '[\\w.]+' : spell([...aliases, ...ambiguous]); + const target = new RegExp(`(? target.test(t.trim())) ?? false; + if (!assigned && !dynamic.test(statement)) continue; + if (test) { out.writers.add(file); continue; } + // A write through an ambiguous spelling may land on another module: unknown. + const single = assignment && assignment.targets.length === 1 && !assignment.augmented && exact.test(assignment.targets[0]!.trim()); + const value = single ? assignment!.value.trim() : ''; + if (value === 'None') continue; + const constructor = value ? pythonConstructorCall(value) : null; + if (!constructor) { out.unknown = true; return out; } + out.writes.push({ type: constructor, file }); + } + } + return out; + }); +} + +/** + * Whether the global's own module can write it through its namespace dict: + * `globals()` / `vars()` / `sys.modules[__name__]` used as anything but a + * literal-key read or `.get`, or a literal-key write of this name. Read per + * statement (continuations joined), with string contents ignored. + */ +function pythonDynamicGlobalWrite(name: string, filePath: string, context: ResolutionContext): boolean { + const lines = pythonMemberLines(filePath, context); + const statements = pythonStatementStarts(filePath, context); + const script = pythonMainBlockLines(filePath, context); + // `vars()` is the module dict only at module scope; inside a function it is the locals. + const moduleLines = new Set(pythonOwnLines(null, filePath, context)); + for (let i = 0; i < lines.length; i++) { + if (!statements[i] || script.has(i)) continue; + let statement = lines[i]!; + for (let j = i + 1; j < lines.length && !statements[j]; j++) statement += '\n' + lines[j]; + const code = blankPythonStrings(statement); + const namespace = moduleLines.has(i) + ? /\bglobals\(\s*\)|\bvars\(\s*\)|\bsys\.modules\s*\[\s*__name__\s*\]/g + : /\bglobals\(\s*\)|\bsys\.modules\s*\[\s*__name__\s*\]/g; + const uses = code.match(namespace)?.length ?? 0; + if (!uses) continue; + const targets = pythonAssignment(statement)?.targets ?? []; + let safe = 0; + for (const m of statement.matchAll(/\bglobals\(\s*\)\s*(?:\[\s*(['"])(\w+)\1\s*\]|\.\s*get\s*\()/g)) { + const key = m[2]; + const written = key !== undefined && targets.some(t => t.includes(m[0])); + if (written && key === name) return true; + safe++; + } + if (uses > safe) return true; + } + return false; +} + +/** + * The method a module global's value can dispatch to: one class resolves to + * its own method; several bind to the nearest declaration they all inherit, + * as a base-typed receiver does. Otherwise, no edge. + */ +function pythonGlobalMembers(global: Node, member: string, ref: UnresolvedRef, context: ResolutionContext): Node[] { + // A test that installs its own double sees the double, not the production type. + if (pythonExternalWrites(global, context).writers.has(ref.filePath)) return []; + const classes = pythonGlobalClasses(global, ref, context); + if (!classes) return []; + const targets = [...new Map(classes.flatMap(cls => pythonMembers(cls, member, ref, context)).map(n => [n.id, n])).values()]; + if (targets.length <= 1) return targets; + // Several targets: the nearest declaration every candidate class inherits, + // whether or not a candidate overrides it. + const owner = (n: Node) => context.getNodesInFile(n.filePath).find(c => + c.kind === 'class' && n.qualifiedName === `${c.qualifiedName}::${member}`); + const declarations = new Map(); + const queue = [...targets]; + while (queue.length && declarations.size < 32) { + const decl = queue.shift()!; + const cls = decl && owner(decl); + if (!cls || declarations.has(decl.id)) continue; + declarations.set(decl.id, { decl, cls }); + queue.push(...pythonBases(cls, ref, context).flatMap(base => pythonMembers(base, member, ref, context))); + } + const inherits = (cls: Node, base: Node) => cls.id === base.id || pythonDerivesFrom(cls, base, ref, context); + const shared = [...declarations.values()].filter(d => classes.every(c => inherits(c, d.cls))); + const nearest = shared.filter(d => shared.every(o => inherits(d.cls, o.cls))); + return nearest.length === 1 ? [nearest[0]!.decl] : targets; +} + +/** `Cls(...)` / `pkg.mod.Cls(...)` as the WHOLE (possibly multi-line) value; else null (`Cls() if x else y`). */ +function pythonConstructorCall(text: string): string | null { + // `(Cls())` is the same value; peel parentheses that wrap the whole expression. + for (let wrapped = text.trim(); wrapped.startsWith('('); ) { + let depth = 0; + let close = -1; + for (let i = 0; i < wrapped.length && close < 0; i++) { + if (wrapped[i] === '(') depth++; + else if (wrapped[i] === ')' && --depth === 0) close = i; + } + if (close !== wrapped.length - 1) break; + text = wrapped = wrapped.slice(1, -1).trim(); + } + const callee = text.match(/^((?:[A-Za-z_]\w*\.)*[A-Z]\w*)\s*\(/); + if (!callee) return null; + let depth = 0; + let quote = ''; + for (let i = callee[0].length - 1; i < text.length; i++) { + const c = text[i]!; + if (quote) { if (c === '\\') i++; else if (c === quote) quote = ''; continue; } + if (c === '"' || c === "'") quote = c; + else if (c === '(' || c === '[' || c === '{') depth++; + else if (c === ')' || c === ']' || c === '}') { + if (--depth === 0) return /^[\s\\]*$/.test(text.slice(i + 1)) ? callee[1]! : null; + } + } + return null; +} + /** Read a field's own annotation/initializer, or a constructor parameter assigned to it. */ function pythonFieldType(receiver: string, owner: Node, ref: UnresolvedRef, context: ResolutionContext): string | null { const lines = pythonMemberLines(ref.filePath, context); @@ -6094,6 +6590,9 @@ function getInferScanStates(context: ResolutionContext): Map