diff --git a/pytype/tools/CMakeLists.txt b/pytype/tools/CMakeLists.txt index 73a8aad47..15ebb3128 100644 --- a/pytype/tools/CMakeLists.txt +++ b/pytype/tools/CMakeLists.txt @@ -106,4 +106,3 @@ add_subdirectory(annotate_ast) add_subdirectory(debugger) add_subdirectory(merge_pyi) add_subdirectory(traces) -add_subdirectory(xref) diff --git a/pytype/tools/xref/CMakeLists.txt b/pytype/tools/xref/CMakeLists.txt deleted file mode 100644 index b431dfdfe..000000000 --- a/pytype/tools/xref/CMakeLists.txt +++ /dev/null @@ -1,148 +0,0 @@ -add_package() - -py_library( - NAME - indexer - DEPS - ._indexer - .debug - .utils - .kythe - .output - .parse_args -) - -toplevel_py_binary( - NAME - xref - SRCS - main.py - MAIN - main.py - DEPS - ._indexer - .debug - .kythe - .output - .parse_args - pytype.utils -) - -py_library( - NAME - callgraph - SRCS - callgraph.py - DEPS - pytype.pytd.pytd -) - -py_library( - NAME - debug - SRCS - debug.py - DEPS - pytype.ast.ast -) - -py_library( - NAME - utils - SRCS - utils.py - DEPS - pytype.utils -) - -py_library( - NAME - _indexer - SRCS - __init__.py - indexer.py - DEPS - .callgraph - .utils - .node_utils - pytype.libvm - pytype.utils - pytype.abstract.abstract - pytype.ast.ast - pytype.pytd.pytd - pytype.tools.traces.traces -) - -py_library( - NAME - kythe - SRCS - kythe.py - DEPS - ._indexer - .utils -) - -py_library( - NAME - node_utils - SRCS - node_utils.py -) - -py_library( - NAME - output - SRCS - output.py -) - -py_library( - NAME - parse_args - SRCS - parse_args.py - DEPS - .kythe - pytype.config - pytype.utils - pytype.tools.tools -) - -py_test( - NAME - callgraph_test - SRCS - callgraph_test.py - DEPS - ._indexer - pytype.config - pytype.tests.test_base -) - -py_test( - NAME - indexer_test - SRCS - indexer_test.py - DEPS - ._indexer - .kythe - .output - pytype.config - pytype.utils - pytype.abstract.abstract - pytype.tests.test_base -) - -py_test( - NAME - parse_args_test - SRCS - parse_args_test.py - DEPS - .parse_args - pytype.tests.test_base -) - -add_subdirectory(testdata) diff --git a/pytype/tools/xref/README.md b/pytype/tools/xref/README.md deleted file mode 100644 index 0d2c64955..000000000 --- a/pytype/tools/xref/README.md +++ /dev/null @@ -1,4 +0,0 @@ -An implementation of cross references for python, built on top of the type -information gathered by pytype. - -This project is integrated with [kythe](https://github.com/kythe/kythe) diff --git a/pytype/tools/xref/TESTING.md b/pytype/tools/xref/TESTING.md deleted file mode 100644 index d1d915696..000000000 --- a/pytype/tools/xref/TESTING.md +++ /dev/null @@ -1,8 +0,0 @@ -The main tests are the kythe verifier tests under testdata/ - -Get the latest [kythe -release](https://github.com/kythe/kythe/releases) and unpack it to `$HOME/kythe`. Then, to run a test (assuming `xref` is in your path): - -```` -xref testdata/testcase.py | ./kythe-verifier.sh testdata/testcase.py -```` diff --git a/pytype/tools/xref/__init__.py b/pytype/tools/xref/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/pytype/tools/xref/callgraph.py b/pytype/tools/xref/callgraph.py deleted file mode 100644 index d521ab9d8..000000000 --- a/pytype/tools/xref/callgraph.py +++ /dev/null @@ -1,208 +0,0 @@ -"""Trace function arguments, return values and calls to other functions.""" - -import dataclasses -from typing import Any - -from pytype.pytd import escape -from pytype.pytd import pytd -from pytype.pytd import pytd_utils - - -@dataclasses.dataclass -class Attr: - name: str - node_type: str - type: Any - attrib: str - location: str - - -@dataclasses.dataclass -class Arg: - name: str - node_type: str - type: Any - - -@dataclasses.dataclass -class Param: - name: str - type: Any - - -@dataclasses.dataclass -class Call: - function_id: str - args: list[Arg] - location: str - - -@dataclasses.dataclass -class Function: - id: str - params: list[Any] = dataclasses.field(default_factory=list) - param_attrs: list[Any] = dataclasses.field(default_factory=list) - local_attrs: list[Any] = dataclasses.field(default_factory=list) - calls: list[Any] = dataclasses.field(default_factory=list) - ret: Any = dataclasses.field(default=None) - location: Any = dataclasses.field(default=None) - - -def unknown_to_any(typename): - if escape.UNKNOWN in typename: - return 'typing.Any' - return typename - - -def unwrap_type(typ): - if isinstance(typ, (pytd.ClassType, pytd.NamedType)): - typ_name = typ.name - elif isinstance(typ, pytd.UnionType): - typ_name = 'Union[' + ', '.join(unwrap_type(t) for t in typ.type_list) + ']' - elif isinstance(typ, pytd.AnythingType): - typ_name = 'typing.Any' - else: - typ_name = pytd_utils.Print(typ) - return unknown_to_any(typ_name) - - -def get_function_params(pytd_fn): - """Collect function param types from pytype.""" - # We have turned call records on in the indexer, so a function will have a - # pytd signature for every tuple of call args. Here we iterate through those - # signatures and set every param's type to the union of every non-"unknown" - # call type for that param. - params = {} - for sig in pytd_fn.signatures: - for p in sig.params: - if p.name not in params: - params[p.name] = [] - if escape.UNKNOWN not in str(p.type): - params[p.name].append(p.type) - for k in params: - params[k] = pytd_utils.JoinTypes(params[k]) - return [(k, unwrap_type(v)) for k, v in params.items()] - - -class FunctionMap: - """Collect a map of function types and outbound callgraph edges.""" - - def __init__(self, index): - self.index = index - self.fmap = self.init_from_index(index) - - def pytd_of_fn(self, f): - """Get the pytype pytd function definition.""" - if f.data and f.data[0]: - d = f.data[0][0] - try: - return self.index.get_pytd_def(d, f.name) - except: # pylint: disable=bare-except - # We sometimes get Instance(PyTDClass(str)) here, which throws an - # exception in get_pytd_def, possibly due to other earlier problems. - # Don't crash the indexer if this happens. - return None - else: - # TODO(mdemello): log this - return None - - def init_from_index(self, index): - """Initialize the function map.""" - out = {} - fn_defs = [(k, v) for k, v in index.defs.items() if v.typ == 'FunctionDef'] - for fn_id, fn in fn_defs: - pytd_fn = self.pytd_of_fn(fn) - if isinstance(pytd_fn, pytd.Function): - params = get_function_params(pytd_fn) - else: - # Sometimes pytype cannot infer the type of a function, and falls back - # to Any. Don't crash the indexer if this happens. - params = [] - params = [Param(name, typ) for name, typ in params] - ret = index.envs[fn_id].ret - if fn_id in index.locs: - location = index.locs[fn_id][-1].location - else: - location = None - out[fn_id] = Function( - id=fn_id, params=params, ret=ret, location=location) - # Add a slot for "module" to record function calls made at top-level - out['module'] = Function(id='module') - return out - - def add_attr(self, ref, defn): - """Add an attr access within a function body.""" - attrib = ref.name - scope = ref.ref_scope - if scope not in self.fmap: - # This call was not within a function body. - return - - try: - d = self.index.envs[scope].env[ref.target] - except KeyError: - return - - typename = unknown_to_any(defn.typename) - attr_access = Attr( - name=d.name, - node_type=d.typ, - type=typename, - attrib=attrib, - location=ref.location) - fn = self.fmap[scope] - if attr_access.node_type == 'Param': - fn.param_attrs.append(attr_access) - else: - fn.local_attrs.append(attr_access) - - def add_param_def(self, ref, defn): - """Add a function parameter definition.""" - fn = self.fmap[ref.ref_scope] - for param in fn.params: - # Don't override a type inferred from call sites. - if param.name == defn.name and param.type in ('nothing', 'typing.Any'): - param.type = unwrap_type(self.index.get_pytd(ref.data[0])) - break - - def add_link(self, ref, defn): - if ref.typ == 'Attribute': - self.add_attr(ref, defn) - if defn.typ == 'Param': - self.add_param_def(ref, defn) - - def add_call(self, call): - """Add a function call.""" - scope = call.scope - if scope not in self.fmap: - # This call was not within a function body. - return - env = self.index.envs[scope] - args = [] - for name in call.args: - if name in env.env: - defn = env.env[name] - node_type = defn.typ - typename = unknown_to_any(defn.typename) - else: - node_type = None - typename = 'typing.Any' - args.append(Arg(name, node_type, typename)) - self.fmap[scope].calls.append( - Call(call.func, args, call.location)) - - -def collect_function_map(index): - """Track types and outgoing calls within a function.""" - - fns = FunctionMap(index) - - # Collect methods and attribute accesses - for ref, defn in index.links: - fns.add_link(ref, defn) - - # Collect function calls - for call in index.calls: - fns.add_call(call) - - return fns.fmap diff --git a/pytype/tools/xref/callgraph_test.py b/pytype/tools/xref/callgraph_test.py deleted file mode 100644 index 089163907..000000000 --- a/pytype/tools/xref/callgraph_test.py +++ /dev/null @@ -1,188 +0,0 @@ -from pytype import config - -from pytype.tests import test_base -from pytype.tests import test_utils -from pytype.tools.xref import indexer - - -@test_base.skip(reason="The callgraph code only works in Python 3.5-6.") -class CallgraphTest(test_base.BaseTest): - """Tests for the callgraph.""" - - def index_code(self, code, **kwargs): - """Generate references from a code string.""" - args = {"version": self.python_version} - args.update(kwargs) - with test_utils.Tempdir() as d: - d.create_file("t.py", code) - options = config.Options.create(d["t.py"]) - options.tweak(**args) - return indexer.process_file(options, generate_callgraphs=True) - - def assertAttrsEqual(self, attrs, expected): - actual = {(x.name, x.type, x.attrib) for x in attrs} - self.assertCountEqual(actual, expected) - - def assertCallsEqual(self, calls, expected): - actual = [] - for c in calls: - actual.append( - (c.function_id, [(a.name, a.node_type, a.type) for a in c.args])) - self.assertCountEqual(actual, expected) - - def assertParamsEqual(self, params, expected): - actual = {(x.name, x.type) for x in params} - self.assertCountEqual(actual, expected) - - def assertHasFunctions(self, fns, expected): - actual = fns.keys() - expected = ["module"] + [f"module.{x}" for x in expected] - self.assertCountEqual(actual, expected) - - def test_basic(self): - ix = self.index_code(""" - def f(x: str): - y = x.strip() - return y - - def g(y): - a = f(y) - b = complex(1, 2) - c = b.real - return c - """) - fns = ix.function_map - self.assertHasFunctions(fns, ["f", "g"]) - f = fns["module.f"] - self.assertAttrsEqual(f.param_attrs, - {("x", "builtins.str", "x.strip")}) - self.assertAttrsEqual(f.local_attrs, set()) - self.assertCallsEqual(f.calls, [("str.strip", [])]) - self.assertEqual(f.ret.id, "module.f.y") - self.assertParamsEqual( - f.params, [("x", "builtins.str")]) - - g = fns["module.g"] - self.assertAttrsEqual(g.param_attrs, set()) - self.assertAttrsEqual(g.local_attrs, - {("b", "builtins.complex", "b.real")}) - self.assertCallsEqual(g.calls, [ - ("f", [("y", "Param", "typing.Any")]), - ("complex", []) - ]) - self.assertEqual(g.ret.id, "module.g.c") - self.assertParamsEqual(g.params, [("y", "typing.Any")]) - - def test_remote(self): - code = """ - import foo - - def f(a, b): - x = foo.X(a) - y = foo.Y(a, b) - z = y.bar() - """ - stub = """ - class X: - def __init__(a: str) -> None: ... - class Y: - def __init__(a: str, b: int) -> None: ... - def bar() -> int: ... - """ - with test_utils.Tempdir() as d: - d.create_file("t.py", code) - d.create_file("foo.pyi", stub) - options = config.Options.create(d["t.py"], pythonpath=d.path, - version=self.python_version) - ix = indexer.process_file(options, generate_callgraphs=True) - fns = ix.function_map - self.assertHasFunctions(fns, ["f"]) - f = fns["module.f"] - self.assertAttrsEqual(f.param_attrs, []) - self.assertAttrsEqual(f.local_attrs, [("y", "foo.Y", "y.bar")]) - self.assertCallsEqual(f.calls, [ - ("X", [("a", "Param", "typing.Any")]), - ("Y", [("a", "Param", "typing.Any"), ("b", "Param", "typing.Any")]), - ("Y.bar", []) - ]) - - def test_no_outgoing_calls(self): - """Capture a function with no outgoing calls.""" - ix = self.index_code(""" - def f(x: int): - return "hello" - """) - fns = ix.function_map - self.assertHasFunctions(fns, ["f"]) - f = fns["module.f"] - self.assertAttrsEqual(f.param_attrs, []) - self.assertAttrsEqual(f.local_attrs, []) - self.assertCallsEqual(f.calls, []) - self.assertParamsEqual(f.params, [("x", "builtins.int")]) - - def test_call_records(self): - """Use a function's call records to infer param types.""" - ix = self.index_code(""" - class A: - def foo(self, x): - return x + "1" - - def f(x, y): - z = x + y - return z - - def g(a): - return f(a, 3) - - def h(b): - y = b - return y - - x = g(10) - y = A() - p = h(y) - q = h("hello") - a = y.foo("1") - """) - fns = ix.function_map - self.assertHasFunctions(fns, ["A.foo", "f", "g", "h"]) - expected = [ - ("f", [("x", "builtins.int"), ("y", "builtins.int")]), - ("g", [("a", "builtins.int")]), - ("h", [("b", "Union[A, builtins.str]")]), - ("A.foo", [("self", "A"), ("x", "builtins.str")]) - ] - for fn, params in expected: - f = fns[f"module.{fn}"] - self.assertParamsEqual(f.params, params) - - def test_toplevel_calls(self): - """Don't index calls outside a function.""" - ix = self.index_code(""" - def f(x: int): - return "hello" - - a = f(10) - a.upcase() - """) - fns = ix.function_map - # we should only have f in fns, despite function calls at module scope - self.assertHasFunctions(fns, ["f"]) - - def test_class_level_calls(self): - """Don't index calls outside a function.""" - ix = self.index_code(""" - def f(x: int): - return "hello" - - class A: - a = f(10) - b = a.upcase() - """) - fns = ix.function_map - # we should only have f in fns, despite function calls at class scope - self.assertHasFunctions(fns, ["f"]) - - -if __name__ == "__main__": - test_base.main() diff --git a/pytype/tools/xref/debug.py b/pytype/tools/xref/debug.py deleted file mode 100644 index 3f186cfdb..000000000 --- a/pytype/tools/xref/debug.py +++ /dev/null @@ -1,152 +0,0 @@ -"""Debug utils for working with the indexer and the AST.""" - -import base64 -import re - -from pytype.ast import debug - -# pylint: disable=protected-access -# We never care about protected access when writing debug code! - - -def format_loc(location): - # location is (line, column) - fmt = "%d:%2d" % location - return fmt.rjust(8) - - -def format_def_with_location(defn, loc): - return f"{format_loc(loc)} | {defn.typ.ljust(15)} {defn.format()}" - - -def format_ref(ref): - return (f"{format_loc(ref.location)} | {ref.typ.ljust(15)} " - f"{ref.scope}.{ref.name}") - - -def format_call(call): - return f"{format_loc(call.location)} | {'Call'.ljust(15)} {call.func}" - - -def typename(node): - return node.__class__.__name__ - - -def show_defs(index): - """Show definitions.""" - for def_id in index.locs: - defn = index.defs[def_id] - for loc in index.locs[def_id]: - print(format_def_with_location(defn, loc.location)) - if defn.doc: - print(" "*28 + str(defn.doc)) - - -def show_refs(index): - """Show references and associated definitions.""" - indent = " : " - for ref, defn in index.links: - print(format_ref(ref)) - if defn: - print(indent, defn.format()) - for loc in index.locs[defn.id]: - print(indent, format_def_with_location(defn, loc.location)) - else: - print(indent, "None") - continue - - -def show_calls(index): - for call in index.calls: - print(format_call(call)) - - -def display_type(data): - """Convert a pytype internal type to a display type.""" - name = "typing.Any" - if data and data[0]: - d = data[0][0] - if d.cls: - name = d.cls.full_name - if name == "unsolveable": - name = "typing.Any" - elif name.startswith("z__pytype_partial"): - name = "" - return name - - -def show_types(index): - """Show inferred types.""" - out = [] - for def_id in index.locs: - defn = index.defs[def_id] - for loc in index.locs[def_id]: - out.append( - (loc.location, defn.typ, defn.name, display_type(defn.data), None)) - for ref, defn in index.links: - if len(ref.data) > 1: - typ = display_type(ref.data[1:]) - link_typ = display_type(ref.data) - else: - typ = display_type(ref.data) - link_typ = None - out.append((ref.location, defn.typ, ref.name, typ, link_typ)) - # Sort by location - for location, category, name, typ, link_typ in sorted( - out, key=lambda x: x[0]): - # Filter out some noise - if (category in ("FunctionDef", "IsInstance") or - typ in ("builtins.module", "__future__._Feature")): - continue - if link_typ: - print( - f"{format_loc(location)} | {name.ljust(35)} {typ} [-> {link_typ}]") - else: - print(f"{format_loc(location)} | {name.ljust(35)} {typ}") - - -def show_index(index): - """Display output in human-readable format.""" - - def separator(): - print("\n--------------------\n") - - show_defs(index) - separator() - show_refs(index) - separator() - show_calls(index) - separator() - - -def show_map(name, mapping): - print("%s: {" % name) - for k, v in mapping.items(): - print(" ", k, v) - print("}") - - -def show_kythe_spans(kythe_graph, src): - """Show kythe spans.""" - - for entry in kythe_graph.entries: - if entry.fact_name in ("/kythe/text", "/kythe/loc/start", "kythe/loc/end"): - continue - sig = entry.source.signature - m = re.fullmatch(r"@(\d+):(\d+)", sig) - if m: - start, end = int(m.group(1)), int(m.group(2)) - text = src[start:end] - else: - start, end = 0, 0 - text = "" - value = "" - if hasattr(entry, "fact_value"): - value = base64.b64decode(entry.fact_value.encode("utf-8")).decode("utf-8") - if start: - print(f"({start}, {end}): {value}: {text}") - - -# reexport AST dumper - -dump = debug.dump diff --git a/pytype/tools/xref/indexer.py b/pytype/tools/xref/indexer.py deleted file mode 100644 index 4352a668b..000000000 --- a/pytype/tools/xref/indexer.py +++ /dev/null @@ -1,1348 +0,0 @@ -"""Generate cross references from a project.""" - -import ast as astlib -import collections -import dataclasses -import re -import textwrap -import types -from typing import Any, TypeVar - -from pytype import analyze -from pytype import config -from pytype import io -from pytype import load_pytd -from pytype import module_utils -from pytype.abstract import abstract -from pytype.ast import visitor as ast_visitor -from pytype.pytd import pytd -from pytype.pytd import pytd_utils -from pytype.pytd import visitors -from pytype.tools.traces import source -from pytype.tools.traces import traces -from pytype.tools.xref import callgraph -from pytype.tools.xref import utils as xref_utils -from pytype.tools.xref import node_utils - -# A mapping of offsets between a node's start position and the symbol being -# defined. e.g. in the declaration "class X" the X is at +6 from the start. -DEF_OFFSETS = { - "ClassDef": 6, # class X - "FunctionDef": 4, # def f -} - - -# Marker for a link to a file rather than a node within the file. -IMPORT_FILE_MARKER = "<__FILE__>" - - -# Marker to capture a pending return value while traversing an AST -_RETURNING_NAME = "RETURNING NAME" - -_T = TypeVar("_T") - - -def qualified_method(data): - """Fully qualify a method call with its class scope.""" - if isinstance(data, abstract.BoundFunction): - return data.repr_names() - else: - return [data.name] - - -def get_location(node): - # TODO(mdemello): The column offset for nodes like "class A" needs to be - # adjusted to the start of the symbol. - return source.Location(node.lineno, node.col_offset) - - -def get_end_location(node): - end_lineno = node.end_lineno - end_col_offset = node.end_col_offset - return source.Location(end_lineno, end_col_offset) - - -def match_opcodes(opcode_traces, lineno, op_match_list): - """Get all opcodes matching op_match_list on a given line. - - Args: - opcode_traces: traces - lineno: line number to get ops from. - op_match_list: [(opcode_name, symbol|None), ...]; None matches any symbol. - - Returns: - A list of matching opcodes. - """ - out = [] - for trace in opcode_traces[lineno]: - for match_op, match_symbol in op_match_list: - if trace.op == match_op and match_symbol in [None, trace.symbol]: - out.append((trace.op, trace.symbol, trace.types)) - return out - - -def match_opcodes_multiline(opcode_traces, start, end, op_match_list): - """Get all opcodes matching op_match_list in a range of lines.""" - out = [] - for line in range(start, end + 1): - out.extend(match_opcodes(opcode_traces, line, op_match_list)) - return out - - -def _unwrap(data): - assert len(data) == 1 - return data[0] - - -# Internal datatypes - - -class AttrError(Exception): - pass - - -@dataclasses.dataclass -class PytypeValue: - """Stores a value inferred by pytype.""" - - module: str - name: str - typ: Any - id: str | None = dataclasses.field(default=None, init=False) - - def __post_init__(self): - self.id = self.module + "." + self.name - - def format(self): - return f"{self.id} {{ {self.module}.{self.typ} : {self.name} }}" - - @classmethod - def _from_data(cls, data): - """Construct a PytypeValue from a single datum.""" - - if isinstance(data, abstract.PyTDClass): - if data.module: - # If we have a remote reference, return Remote rather than PytypeValue. - return Remote(data.module, data.name, resolved=True) - else: - # This is a namedtuple or some other special case pytype has generated a - # local PyTDClass for. We need special cases for them too. - return None - elif isinstance(data, abstract.Module): - return Remote(data.name, IMPORT_FILE_MARKER, resolved=True) - elif isinstance(data, abstract.InterpreterClass): - return cls("module", data.name, "Class") - elif isinstance(data, abstract.BoundFunction): - # TODO(mdemello): Handle multiple class bindings. - name = data.repr_names(callself_repr=node_utils.typename)[0] - return cls("module", name, "BoundFunction") - else: - # TODO(mdemello): We need to infer the module here. - return cls("module", str(data), node_utils.typename(data)) - - @classmethod - def from_data(cls, data): - """Construct a PytypeValue from a list of data.""" - - if not data: - return None - else: - return [cls._from_data(x) for x in data] - - def to_signature(self): - return self.module + "." + self.name - - @property - def typename(self): - return self.to_signature() - - -@dataclasses.dataclass -class Module: - """Module representation.""" - name: str - - def attr(self, attr_name): - return Remote(self.name, attr_name, resolved=True) - - def submodule(self, attr_name): - name = self.name + "." + attr_name - return Remote(name, IMPORT_FILE_MARKER, resolved=True) - - -@dataclasses.dataclass -class DocString: - """Store the text and location of a docstring.""" - - text: str - location: source.Location - length: int - - @classmethod - def from_node(cls: type[_T], ast: types.ModuleType, node) -> _T | None: - """If the first element in node.body is a string, create a docstring.""" - - # This should only be called on ClassDef and FunctionDef - assert isinstance(node, (ast.ClassDef, ast.FunctionDef)) - if (node.body and - isinstance(node.body[0], ast.Expr) and - isinstance(node.body[0].value, ast.Constant) and - isinstance(node.body[0].value.value, str)): - doc_node = node.body[0] - doc = doc_node.value.value - length = len(doc) # we want to preserve the byte length - # strip indentation from multiline docstrings - if "\n" in doc: - first, rest = doc.split("\n", 1) - doc = first + "\n" + textwrap.dedent(rest).rstrip() - return cls(doc, get_location(doc_node), length) - return None - - -@dataclasses.dataclass -class Definition: - """A symbol definition. - - Attributes: - name: The symbol name - typ: The definition type (e.g. ClassDef) - data: Pytype data from the opcode traces - scope: The namespace id (e.g. module:class A:function f:x) - target: The LHS of an attribute (e.g. for x.foo, target = typeof(x)) - doc: The docstring, if any, for function and class defs - id: The id - """ - - name: str - typ: Any - data: Any - scope: str - target: Any - doc: str | None - id: str | None = dataclasses.field(default=None, init=False) - - def __post_init__(self): - self.id = self.scope + "." + self.name - - def format(self): - return self.id - - def to_signature(self): - return self.id - - def doc_signature(self): - """Signature for the definition's docstring.""" - return self.to_signature() + ".__doc__" - - def node_kind(self): - # TODO(mdemello): Add more node types. - if self.typ == "ClassDef": - return "class" - elif self.typ == "FunctionDef": - return "function" - else: - return "variable" - - def subkind(self) -> str | None: - if self.typ == "Import" or self.typ == "ImportFrom": - return "import" - return None - - @property - def typename(self): - """The fully qualified type of the object the definition is bound to.""" - if self.data and self.data[0]: - d = self.data[0][0] - if d.cls: - return d.cls.full_name - else: - return "typing.Any" - else: - return "typing.Any" - - -@dataclasses.dataclass -class Remote: - """A symbol from another module.""" - - module: str - name: str - resolved: bool - id: str | None = dataclasses.field(default=None, init=False) - typ: Any = dataclasses.field(default=None, init=False) - - def __post_init__(self): - self.id = self.module + "/module." + self.name - - def attr(self, attr_name): - return Remote(self.module, self.name + "." + attr_name, self.resolved) - - def format(self): - return self.id - - @property - def typename(self): - name = self.name.split(".", 1)[0] - return self.module + "." + name - - -@dataclasses.dataclass -class DefLocation: - """A location of a symbol definition. - - Attributes: - def_id: The definition id (scope + name) - location: The location of the definition in the source code. - - Note that a single definition can have multiple locations, for symbols that - are redefined in the code. - """ - - def_id: str - location: source.Location - - -@dataclasses.dataclass -class FunctionParam: - """A link between a function def and the defs of its params.""" - - def_id: str - param_id: str - position: int - - -@dataclasses.dataclass -class Reference: - """A symbol holding a reference to a definition. - - Attributes: - name: The symbol name - typ: The symbol type (e.g. Attribute) - data: The pytype data attached to the symbol - scope: The namespace id (e.g. module.A.f) - ref_scope: The namespace id of the referred symbol (if we can determine it) - target: The LHS of an attribute (e.g. for x.foo, target = typeof(x)) - location: The line and column of the symbol in the source code - id: The id - """ - - name: str - typ: Any - data: Any - scope: str - ref_scope: str | None - target: Any - location: source.Location - id: str | None = dataclasses.field(default=None, init=False) - - def __post_init__(self): - self.id = self.scope + "." + self.name - - def format(self): - return self.id - - -@dataclasses.dataclass -class NameArg: - """Representation of a single-variable function call argument.""" - - name: str - type: Any - - -@dataclasses.dataclass -class ExprArg: - """Representation of an expression function call argument.""" - - names: list[str] - type: Any - - -@dataclasses.dataclass -class Funcall: - """Representation of a function call.""" - - name: str - scope: str - func: str - location: source.Location - end_location: source.Location - args: list[Any] - return_type: str - - -class Env: - """A collection of namespaced symbols.""" - - def __init__(self, ast, scope, parent, cls): - """Initialize an environment. - - Arguments: - ast: An ast module - scope: The namespace key (e.g. module:class A:function f) - parent: The env of the directly enclosing namespace - cls: The class currently being defined - (None if we are not in a class definition) - - Other attributes defined: - env: The dictionary holding the symbol table for this environment - attrs: Attributes defined on the current class - self_var: The `self` variable in method definitions - ret: The `return` variable for functions - """ - - self.ast = ast - self.scope = scope - self.parent = parent - self.cls = cls - self.env = {} - self.attrs = None - self.self_var = parent and parent.self_var - self.ret = None - - def lookup(self, symbol): - if symbol in self.env: - return (self, self.env[symbol]) - elif self.parent: - return self.parent.lookup(symbol) - else: - return (None, None) - - def __getitem__(self, symbol): - return self.lookup(symbol)[1] - - def __setitem__(self, symbol, value): - self.env[symbol] = value - - def is_self_attr(self, node): - if not self.self_var or not isinstance(node, self.ast.Attribute): - return False - if isinstance(node.value, self.ast.Name): - name = node.value.id - else: - name = node.value - return name == self.self_var.name - - def getattr(self, attrib): - if self.attrs is not None and attrib in self.attrs: - return self.attrs[attrib] - elif self.cls and self.cls.scope != self.scope: - return self.cls.getattr(attrib) - else: - raise AttrError("called getattr in non-class context") - - def setattr(self, attrib, value): - if self.attrs is not None: - self.attrs[attrib] = value - elif self.cls is not None: - return self.cls.setattr(attrib, value) - else: - raise AttrError("called setattr in non-class context") - - -# pylint: disable=invalid-name -# pylint: disable=missing-docstring -# -# Visitors use generated method names that don't follow the pylint spec. -# Also names like visit_Name are self-documenting and do not need docstrings. - - -class ScopedVisitor(ast_visitor.BaseVisitor): - """An AST node visitor that keeps track of scopes and environments. - - A "scope" is the abstract namespace (represented by a string key that tracks - the nested path of namespaces from the module root, e.g. module:class A:f). - An "environment" holds data for the current scope. self.envs is not - hierarchical, it's just a flat mapping of scope keys to environments. - """ - - # TODO(mdemello): Is the two-level visitor hierarchy really buying us - # anything by way of maintainability or readability? - - def __init__(self, ast, module_name, **kwargs): - super().__init__(ast=ast, **kwargs) - self.stack = [] - self.class_ids = [] - self.envs = {} - self.module_name = module_name - - def get_id(self, node): - """Construct an id based on node type.""" - - c = node.__class__ - if c == self._ast.FunctionDef: - return node.name - elif c == self._ast.ClassDef: - return node.name - elif c == self._ast.Module: - return self.module_name - else: - raise Exception(f"Unexpected scope: {node!r}") # pylint: disable=broad-exception-raised - - def iprint(self, x): - """Print messages indented by scope level, for debugging.""" - print(" " * len(self.stack), x) - - def scope_id(self): - return ".".join(self.get_id(x) for x in self.stack) - - @property - def current_class(self): - if self.class_ids: - return self.envs[self.class_ids[-1]] - return None - - @property - def current_env(self): - current_scope = self.scope_id() - return self.envs[current_scope] - - def add_scope(self, node, is_class=False): - if self.stack: - parent = self.current_env - else: - parent = None - self.stack.append(node) - new_scope = self.scope_id() - new_env = Env(ast=self._ast, - scope=new_scope, - parent=parent, - cls=self.current_class) - if is_class: - new_env.attrs = {} - self.envs[new_scope] = new_env - return new_env - - def enter_ClassDef(self, node): - new_env = self.add_scope(node, is_class=True) - self.class_ids.append(self.scope_id()) - # We need to set the env's cls to the new class, not the enclosing one. - new_env.cls = self.current_class - - def leave_ClassDef(self, _): - self.class_ids.pop() - - def enter_FunctionDef(self, node): - self.add_scope(node) - - def enter_Module(self, node): - super().enter_Module(node) # pytype: disable=attribute-error - self.add_scope(node) - - def leave(self, node): - """If the node has introduced a new scope, we need to pop it off.""" - super().leave(node) - if node == self.stack[-1]: - self.stack.pop() - - -class IndexVisitor(ScopedVisitor, traces.MatchAstVisitor): - """Visitor that generates indexes.""" - - def __init__(self, ast, src, module_name): - super().__init__(ast=ast, src_code=src, module_name=module_name) - self.defs = {} - self.locs = collections.defaultdict(list) - self.refs = [] - self.modules = {} - self.aliases = {} - self.source = src - self.traces = src.traces - self.typemap = {} - self.classmap = {} - self.calls = [] - self.function_params = [] - # Record childof relationships for nested functions/classes - self.childof = [] - self.scope_defn = {} - - def _get_location(self, node, args): - """Get a more accurate node location.""" - - loc = None - - if isinstance(node, self._ast.ClassDef): - # For class and function definitions, search for the string - # (class|def) - # between the start of the AST node and the start of the body. Handles the - # offset for decorated functions/classes. - body_start = node.body[0].lineno - text = f"class {args['name']}" - loc = self.source.find_first_text(node.lineno, body_start, text) - elif isinstance(node, self._ast.FunctionDef): - body_start = node.body[0].lineno - text = f"def {args['name']}" - loc = self.source.find_first_text(node.lineno, body_start, text) - - if loc is None: - loc = get_location(node) - - return loc - - def _get_node_name(self, node): - if isinstance(node, str): - # We replace nodes with their names after visiting them. - return node - return super()._get_node_name(node) - - def make_def(self, node, **kwargs): - """Make a definition from a node.""" - - if isinstance(node, self._ast.Name): - t = node_utils.typename(node.ctx) - elif isinstance(node, self._ast.arg): - t = "Param" - else: - t = node_utils.typename(node) - args = { - "name": node_utils.get_name(node, self._ast), - "scope": self.scope_id(), - "typ": t, - "data": None, - "target": None, - "doc": None, - } - args.update(kwargs) - defn = Definition(**args) - line, col = self._get_location(node, args) - assert line is not None - defloc = DefLocation(defn.id, source.Location(line, col)) - return (defn, defloc) - - def make_ref(self, node, **kwargs): - """Make a reference from a node.""" - - assert "data" in kwargs # required kwarg - args = { - "name": node_utils.get_name(node, self._ast), - "scope": self.scope_id(), - "ref_scope": None, - "typ": node_utils.typename(node), - "location": get_location(node), - "target": None, - } - args.update(kwargs) - return Reference(**args) - - def add_local_def(self, node, **kwargs): - defn, defloc = self.make_def(node, **kwargs) - if defn.id not in self.defs: - self.defs[defn.id] = defn - self.locs[defn.id].append(defloc) - self.envs[defn.scope][defn.name] = defn - return defn - - def add_global_def(self, node, **kwargs): - kwargs.update({"scope": "module"}) - return self.add_local_def(node, **kwargs) - - def add_local_ref(self, node, **kwargs): - kwargs.update({"ref_scope": self.scope_id()}) - ref = self.make_ref(node, **kwargs) - self.refs.append(ref) - return ref - - def add_closure_ref(self, node, **kwargs): - """Look for node.name up the chain of scopes.""" - name = node_utils.get_name(node, self._ast) - env, _ = self.current_env.lookup(name) - if env: - kwargs.update({"ref_scope": env.scope}) - else: - # This should never happen! If python has generated a LOAD_DEREF bytecode - # then we do have the name defined in a parent scope. However, in the - # interests of not crashing the indexer, fall back to the current scope. - # TODO(mdemello): We need error logs. - pass - ref = self.make_ref(node, **kwargs) - self.refs.append(ref) - return ref - - def add_global_ref(self, node, **kwargs): - kwargs.update({"ref_scope": "module"}) - return self.add_local_ref(node, **kwargs) - - def add_call(self, node, name, func, arg_varnames, return_type): - start = get_location(node) - end = get_end_location(node) - self.calls.append( - Funcall(name, self.scope_id(), func, start, end, arg_varnames, - return_type)) - - def add_attr(self, node): - defn, _ = self.make_def(node) - self.defs[defn.id] = defn - env = self.envs[self.scope_id()] - if env.is_self_attr(node): - self.envs[self.scope_id()].setattr(node.attr, defn) - - def _has_decorator(self, f, decorator): - for d in f.decorator_list: - if isinstance(d, self._ast.Name) and d.id == decorator: - return True - return False - - def _record_childof(self, node, defn): - """Record a childof relationship for nested definitions.""" - parent = self.scope_defn.get(self.scope_id()) - if parent: - self.childof.append((defn, parent)) - - def enter_ClassDef(self, node): - class_name = node_utils.get_name(node, self._ast) - last_line = max(node.lineno, node.body[0].lineno - 1) - - ops = match_opcodes_multiline(self.traces, node.lineno, last_line, [ - ("LOAD_BUILD_CLASS", None), - ("STORE_NAME", class_name), - # Classes defined within a function generate a STORE_FAST or - # STORE_DEREF op. - ("STORE_FAST", class_name), - ("STORE_DEREF", class_name), - # A class being declared global anywhere generates a STORE_GLOBAL op. - ("STORE_GLOBAL", class_name), - ]) - # pytype sometimes analyses this twice, leading to duplicate opcode - # traces. We only want the first two in the list. - d = data = None - if (len(ops) >= 2 and - ops[0][0] == "LOAD_BUILD_CLASS" and - ops[1][0] in ( - "STORE_NAME", "STORE_FAST", "STORE_DEREF", "STORE_GLOBAL")): - _, _, data = ops[1] - d = _unwrap(data) - - assert d, "Did not get pytype data for class %s at line %d" % ( - class_name, node.lineno) - defn = self.add_local_def(node, data=data, - doc=DocString.from_node(self._ast, node)) - self._record_childof(node, defn) - self.classmap[d[0]] = defn - super().enter_ClassDef(node) - self.scope_defn[self.scope_id()] = defn - - def enter_FunctionDef(self, node): - last_line = max(node.lineno, node.body[0].lineno - 1) - ops = match_opcodes_multiline(self.traces, node.lineno, last_line, [ - ("MAKE_FUNCTION", None), # py2 has no symbol, py3 has node.name - ("LOAD_CLOSURE", None) # Nested functions - ]) - if ops: - _, _, data = ops[0] - else: - # TODO(mdemello): Add an assert; this should not happen but I would rather - # not break grok indexing if it does. - data = None - fn_def = self.add_local_def(node, data=data, - doc=DocString.from_node(self._ast, node)) - self._record_childof(node, fn_def) - env = self.add_scope(node) - self.scope_defn[self.scope_id()] = fn_def - # TODO(mdemello): Get pytype data for params - args = node.args - posonlyargs = getattr(args, "posonlyargs", []) - vararg = [args.vararg] if getattr(args, "vararg", None) else [] - kwarg = [args.kwarg] if getattr(args, "kwarg", None) else [] - all_args = posonlyargs + args.args + vararg + args.kwonlyargs + kwarg - params = [self.add_local_def(v) for v in all_args] - for i, param in enumerate(params): - self.function_params.append(FunctionParam( - def_id=fn_def.id, param_id=param.id, position=i)) - if env.cls: - if (not self._has_decorator(node, "classmethod") and - not self._has_decorator(node, "staticmethod")): - # Don't crash if we have buggy code like - # class A(): def f(): ... - if params: - env.self_var = params[0] - - def visit_Name(self, node): - # We ignore the location returned by match() because we'll recompute the - # same location anyways. - # We use pytype trace data to distinguish between local and global - # variables. - for unused_loc, trace in self.match(node): - op = trace.op - symbol = trace.symbol - data = trace.types - d = _unwrap(data) - ref = None - if op == "LOAD_GLOBAL": - ref = self.add_global_ref(node, name=symbol, data=data) - self.typemap[ref.id] = d - elif op in ["LOAD_FAST", "LOAD_NAME"]: - ref = self.add_local_ref(node, name=symbol, data=data) - self.typemap[ref.id] = d - elif op in ["LOAD_DEREF"]: - ref = self.add_closure_ref(node, name=symbol, data=data) - self.typemap[ref.id] = d - elif op == "STORE_GLOBAL": - defn = self.add_global_def(node, name=symbol, data=data) - self.typemap[defn.id] = d - elif op in ["STORE_FAST", "STORE_NAME", "STORE_DEREF"]: - defn = self.add_local_def(node, name=symbol, data=data) - self.typemap[defn.id] = d - if ref and self.current_env.ret == _RETURNING_NAME: - self.current_env.ret = ref - - return node.id - - def visit_Call(self, node): - name = self._get_node_name(node) - # We have replaced Name() in args with the corresponding string - arg_varnames = [x for x in node.args if isinstance(x, str)] - seen = set() - for _, trace in self.match(node): - call, return_type = trace.types - if call is None: - continue - for d in call: - for f in qualified_method(d): - if f not in seen: - self.add_call(node, name, f, arg_varnames, return_type) - seen.add(f) - return name - - def visit_Assign(self, node): - for v in node.targets: - if isinstance(v, self._ast.Attribute): - self.add_attr(v) - - def visit_AnnAssign(self, node): - parent = self.scope_defn.get(self.scope_id()) - if parent and parent.typ == "ClassDef": - self.add_local_def(node, name=node.target) - - def _add_attr_ref(self, node, node_str, trace): - ref = self.add_local_ref( - node, - target=node.value, - name=node_str, - data=trace.types) - if len(trace.types) == 2: - _, rhs = trace.types - self.typemap[ref.id] = rhs - - def visit_Attribute(self, node): - node_str = self._get_node_name(node) - # match() returns the location of the attribute, whereas the indexer needs - # the location of the value on which the attribute is accessed, in order to - # link function calls. We'll manually adjust the location later. - for unused_loc, trace in self.match(node): - if trace.op in ("LOAD_ATTR", "LOAD_METHOD"): - self._add_attr_ref(node, node_str, trace) - elif trace.op == "STORE_ATTR": - env = self.envs[self.scope_id()] - if env.is_self_attr(node): - # Add a new definition for `self.x = ...` - defn = self.add_local_def(node) - if self.current_class: - # We only support attr definitions within a class definition. - self.current_env.setattr(node.attr, defn) - else: - # Otherwise just add a reference - self._add_attr_ref(node, node_str, trace) - return node_str - - def visit_Subscript(self, node): - return node.value - - def visit_DictComp(self, _node): - return "" - - def visit_ListComp(self, _node): - return "" - - def process_import(self, node): - """Common code for Import and ImportFrom.""" - - for alias, (loc, trace) in zip(node.names, self.match(node)): - # If an import is aliased, match() returns only the symbol/loc of - # the alias, whereas the indexer also needs access to the unaliased - # name in order to reference the imported module. - op = trace.op - symbol = trace.symbol - data = trace.types - defn: Definition | None = None - if alias.asname: - defn = self.add_local_def( - node, name=symbol, target=alias.name, data=data) - defloc = self.locs[defn.id].pop() - self.locs[defn.id].append(DefLocation(defloc.def_id, loc)) - - # Shift symbol/loc back to the unaliased name. - symbol = alias.name - m = re.search("[ ,]" + symbol + r"\b", self.source.line(loc.line)) - if m is None: - # TODO(slebedev): Support multi-line from-imports. - continue - c, _ = m.span() - loc = source.Location(loc.line, c + 1) - - imported = None - try: - [imported] = _unwrap(data) - except (TypeError, ValueError): - resolved = False - else: - resolved = not isinstance(imported, abstract.Unsolvable) - - if not resolved: - continue - - if op == "STORE_NAME": - # for |import x.y as z| or |from x import y as z| we want {z: x.y} - self.add_local_ref(node, name=symbol, data=data, location=loc) - if not isinstance(imported, abstract.Module): - # Make the from-imported symbol available in the current namespace. - remote = Remote(imported.module, name=symbol, resolved=True) - if defn: - self.aliases[defn.id] = remote - self.current_env[symbol] = remote - self.typemap[remote.id] = [imported] - continue - - if defn: - remote = Remote(imported.full_name, IMPORT_FILE_MARKER, resolved=True) - self.aliases[defn.id] = remote - self.modules[defn.id] = imported.full_name - else: - self.modules[self.scope_id() + "." + symbol] = imported.full_name - elif op == "IMPORT_NAME": - # |import x.y| puts both {x: x} and {x.y: x.y} in modules - self.add_local_ref(node, name=symbol, data=data, location=loc) - # TODO(slebedev): Reference every import path component. - # For example here - # - # from foo.bar import boo - # import foo.bar.boo - # - # we should reference both foo and foo.bar (in addition to foo.bar.boo). - for mod in module_utils.get_all_prefixes(symbol): - self.modules[self.scope_id() + "." + mod] = mod - - def visit_Import(self, node): - self.process_import(node) - - def visit_ImportFrom(self, node): - self.process_import(node) - - def enter_Return(self, node): - if isinstance(node.value, self._ast.Name): - self.current_env.ret = _RETURNING_NAME - - def leave_Return(self, node): - if self.current_env.ret == _RETURNING_NAME: - self.current_env.ret = None - - -# pylint: enable=invalid-name -# pylint: enable=missing-docstring - - -class Indexer: - """Runs the indexer visitor and collects its results.""" - - def __init__(self, - *, - ast, - src, - loader, - pytd_module, - module_name): - self.ast = ast - self.source = src - self.loader = loader - self.pytd_module = pytd_module - self.resolved_modules = loader.get_resolved_modules() - self.imports = xref_utils.process_imports_map(loader.options.imports_map) - self.module_name = module_name - self.traces = src.traces - self.defs = None - self.locs = None - self.refs = None - self.envs = None - self.modules = None - self.aliases = None - self.typemap = None - self.classmap = None - self.calls = None - self.links = [] - self.function_params = None - self.function_map = None - self.childof = [] - - # Optionally preserve the pytype vm so we can access the types later - self.vm = None - - def index(self, code_ast): - """Index an AST corresponding to self.source.""" - - v = IndexVisitor(self.ast, self.source, self.module_name) - v.visit(code_ast) - self.defs = v.defs - self.locs = v.locs - self.refs = v.refs - self.envs = v.envs - self.modules = v.modules - self.aliases = v.aliases - self.typemap = v.typemap - self.classmap = v.classmap - self.calls = v.calls - self.function_params = v.function_params - self.childof = v.childof - - def get_def_offsets(self, defloc): - """Get the byte offsets for a definition.""" - - assert self.defs is not None - defn = self.defs[defloc.def_id] - typ = defn.typ - if typ == "Attribute": - start, end = self._get_attr_bounds(defn.name, defloc.location) - else: - start = self.source.get_offset(defloc.location) - if typ in DEF_OFFSETS: - start += DEF_OFFSETS[typ] - end = start + len(defn.name) - return (start, end) - - def get_doc_offsets(self, doc): - """Get the byte offsets for a docstring.""" - - start = self.source.get_offset(doc.location) - end = start + doc.length - return (start, end) - - def finalize(self): - """Postprocess the information gathered by the tree visitor.""" - self.links = self._lookup_refs() - - def _get_attr_bounds(self, name, location): - """Calculate the anchor bounds for an attr access.""" - return self.get_anchor_bounds( - *self.source.get_attr_location(name, location)) - - def get_anchor_bounds(self, location, length): - """Generate byte offsets from a location and length.""" - - start = self.source.get_offset(location) - end = start + length - return (start, end) - - def get_ref_bounds(self, ref): - if ref.typ == "Attribute": - return self._get_attr_bounds(ref.name, ref.location) - else: - return self.get_anchor_bounds(ref.location, len(ref.name)) - - def _lookup_remote_symbol(self, defn, attr_name): - """Try to look up a definition in an imported module.""" - assert self.modules is not None - if defn.id in self.modules: - return Remote(self.modules[defn.id], name=attr_name, resolved=True) - - if not (defn.typ == "Import" or defn.typ == "ImportFrom"): - return None - - try: - [imported] = _unwrap(defn.data) - except (TypeError, ValueError): - resolved = False - else: - resolved = not isinstance(imported, abstract.Unsolvable) - - if not resolved: - return Remote(defn.name, name=attr_name, resolved=False) - - assert not isinstance(imported, abstract.Module) - assert defn.target - remote = Remote(imported.module, name=defn.target, resolved=True) - return remote.attr(attr_name) - - def _lookup_class_attr(self, name, attrib): - """Look up a class attribute in the environment.""" - - assert self.envs is not None - env = self.envs["module"] - if name not in env.env: - return None - d = env.env[name] - class_env = self.envs[d.id] - _, defn = class_env.lookup(attrib) - return defn - - def _get_attribute_class(self, obj): - """Look up the class of an attribute target.""" - - if isinstance(obj, abstract.Module): - return Module(obj.name) - elif isinstance(obj, abstract.InterpreterClass): - assert self.classmap is not None - return self.classmap.get(obj) - elif isinstance(obj, abstract.PyTDClass): - if obj.module: - return Remote(obj.module, obj.name, resolved=True) - else: - # Corner case: a namedtuple in the MRO of a class will generate a - # PyTDClass even though it's in the current module. - # TODO(mdemello): We need special handling for namedtuples to generate - # and populate a class. - return None - else: - return None - - def _get_mro(self, obj): - if isinstance(obj, abstract.InterpreterClass): - return obj.mro - elif isinstance(obj, abstract.Instance): - return obj.cls.mro - else: - return [] - - def _is_pytype_module(self, obj): - return isinstance(obj, abstract.Module) - - def _lookup_attribute_by_type(self, r, attr_name): - """Look up an attribute using pytype annotations.""" - - links = [] - # For unclear reasons, we sometimes do not get two elements in r.data - # See b/233390756 for an example. This handles the error gracefully, since - # rhs is not used in most of this function. - if not r.data: - return [] - elif len(r.data) == 1: - lhs, rhs = r.data[0], None - else: - lhs, rhs = r.data - - for l in lhs: - if self._is_pytype_module(l): - lookup = [l] - else: - lookup = self._get_mro(l) - for pytype_cls in lookup: - cls = self._get_attribute_class(pytype_cls) - if cls: - if isinstance(cls, Definition): - assert self.envs is not None - env = self.envs[cls.id] - _, attr_value = env.lookup(attr_name) - if not attr_value and isinstance(l, abstract.Instance): - try: - attr_value = env.getattr(attr_name) - except AttrError: - # We will walk up the MRO if we can't find anything. - continue - if attr_value: - links.append((r, attr_value)) - break - elif isinstance(cls, Module): - # Probably extra-cautious about rhs not being a single binding, but - # better to check than crash here. - if len(rhs) == 1 and self._is_pytype_module(rhs[0]): - # e.g. import x.y; a = x.y - links.append((r, cls.submodule(attr_name))) - else: - links.append((r, cls.attr(attr_name))) - break - elif isinstance(cls, Remote): - links.append((r, cls.attr(attr_name))) - break - return links - - def _lookup_refs(self): - """Look up references to generate links.""" - - links = [] - - assert self.refs is not None - for r in self.refs: - if r.typ == "Attribute": - attr_name = r.name.rsplit(".", 1)[-1] - defs = self._lookup_attribute_by_type(r, attr_name) - if defs: - links.extend(defs) - continue - else: - assert self.envs is not None - env = self.envs[r.scope] - env, defn = env.lookup(r.target) - if defn: - # See if this is a definition from an imported module first. - remote = self._lookup_remote_symbol(defn, attr_name) - if remote: - links.append((r, remote)) - else: - # See if we can figure out the class of a bound attribute from the - # typemap. - typ = self.typemap.get(defn.id) - if typ: - for x in PytypeValue.from_data(typ): - if x is None: - continue # Not-yet-special-cased type, e.g. namedtuple. - elif isinstance(x, Remote): - links.append((r, x.attr(attr_name))) - elif x.typ == "Class": - d = self._lookup_class_attr(x.name, attr_name) - if d: - links.append((r, d)) - else: - # Fall back to . - links.append((r, x)) - else: - links.append((r, x)) - else: - links.append((r, defn)) - elif r.typ == "Import" or r.typ == "ImportFrom": - [imported] = _unwrap(r.data) - if isinstance(imported, abstract.Module): - name = IMPORT_FILE_MARKER - if r.name in self.resolved_modules: - module = r.name - else: - module = imported.full_name - else: - assert imported.module - name = r.name - module = imported.module - - links.append((r, Remote(module, name=name, resolved=True))) - else: - try: - assert self.envs is not None - env, defn = self.envs[r.scope].lookup(r.name) - except KeyError: - env, defn = None, None - - if defn: - links.append((r, defn)) - else: - data = PytypeValue.from_data(_unwrap(r.data)) - if data: - for x in data: - links.append((r, x)) - - return links - - def get_pytd_def(self, data, name): - assert self.vm, "Indexer vm has not been preserved." - node = self.vm.ctx.root_node - return self.vm.ctx.pytd_convert.value_to_pytd_def(node, data, name) - - def get_pytd(self, datum): - if not datum: - return pytd.AnythingType() - t = pytd_utils.JoinTypes(v.to_pytd_type() for v in datum).Visit( - visitors.RemoveUnknownClasses()) - return self.loader.resolve_pytd(t, self.pytd_module) - - def make_serializable(self): - """Delete all data that cannot be pickled.""" - for r in self.refs: - r.target = None - r.data = None - - assert self.defs is not None - for d in self.defs.values(): - d.data = None - - for call in self.calls: - call.return_type = None - - assert self.aliases is not None - for a in self.aliases.values(): - a.data = None - - for r, d in self.links: - r.data = None - d.data = None - - # This is all internal data used when building up the final index; if any of - # it proves to be needed we can look at just stripping out the non-picklable - # bits, or converting it to a final form in finalize() - self.ast = None - self.pytd_module = None - self.source = None - self.loader = None - self.resolved_modules = None - self.imports = None - self.traces = None - self.modules = None - self.typemap = None - self.classmap = None - self.envs = None - self.function_params = None - self.calls = None - - -class PytypeError(Exception): - """Wrap exceptions raised by the indexer.""" - - -class VmTrace(source.AbstractTrace): - - def __repr__(self): - types_repr = tuple( - t and [node_utils.typename(x) for x in t] - for t in self.types) - return f"{super().__repr__()} {types_repr}" - - -def process_file(options, source_text=None, generate_callgraphs=False, - preserve_pytype_vm=False): - """Process a single file and return cross references. - - Args: - options: A dictionary of pytype options. - source_text: Optional text of the file; will be read from the file pointed - to by options.input if not supplied. - generate_callgraphs: Collect call graph information - preserve_pytype_vm: Preserve the pytype vm in the indexer - - Returns: - The Indexer object used for indexing. - - Raises: - PytypeError if pytype fails. - """ - with config.verbosity_from(options): - loader = load_pytd.create_loader(options) - src = source_text or io.read_source_file(options.input) - with io.wrap_pytype_exceptions(PytypeError, filename=options.input): - ret = analyze.infer_types( - src=src, - options=options, - loader=loader) - pytd_module = ret.ast - # pylint: disable=unexpected-keyword-arg - ast_root_node = astlib.parse(src, options.input, - feature_version=options.python_version[1]) - # pylint: enable=unexpected-keyword-arg - - # TODO(mdemello): Get from args - module_name = "module" - src_code = source.Code( - src, ret.context.vm.opcode_traces, VmTrace, filename=options.input) - ix = Indexer( - ast=astlib, - src=src_code, - loader=loader, - module_name=module_name, - pytd_module=pytd_module) - ix.index(ast_root_node) - ix.finalize() - - # Make the vm available via indexer.vm for post-finalize() functions. - ix.vm = ret.context.vm - - # Use the indexer as a single object to hold data for calling processes. - if generate_callgraphs: - ix.function_map = callgraph.collect_function_map(ix) - - # Release the vm before returning - if not preserve_pytype_vm: - ix.vm = None - - return ix diff --git a/pytype/tools/xref/indexer_test.py b/pytype/tools/xref/indexer_test.py deleted file mode 100644 index 6c84f9c3c..000000000 --- a/pytype/tools/xref/indexer_test.py +++ /dev/null @@ -1,310 +0,0 @@ -import json -import textwrap - -from pytype import config -from pytype import file_utils -from pytype.abstract import abstract -from pytype.tests import test_base -from pytype.tests import test_utils - -from pytype.tools.xref import indexer -from pytype.tools.xref import kythe -from pytype.tools.xref import output - - -class IndexerTestMixin: - """Mixin for indexer tests.""" - - def index_code(self, code, **kwargs): - """Generate references from a code string.""" - args = {"version": self.python_version} - args.update(kwargs) - with test_utils.Tempdir() as d: - d.create_file("t.py", code) - options = config.Options.create(d["t.py"]) - options.tweak(**args) - return indexer.process_file(options, preserve_pytype_vm=True) - - def generate_kythe(self, code, **kwargs): - """Generate a kythe index from a code string.""" - with test_utils.Tempdir() as d: - d.create_file("t.py", code) - options = config.Options.create(d["t.py"]) - options.tweak(pythonpath=[d.path], version=self.python_version) - k_args = {k: k for k in ["corpus", "root", "path"]} - k_args.update(kwargs) - kythe_args = kythe.Args(**k_args) - ix = indexer.process_file(options) - kg = kythe.generate_graph(ix, kythe_args) - # Collect all the references from the kythe graph. - kythe_index = [json.loads(x) for x in output.json_kythe_graph(kg)] - return kythe_index - - def assertAlias(self, index, fqname, target): - self.assertIn(fqname, index.aliases) - alias = index.aliases[fqname] - self.assertIsInstance(alias, indexer.Remote) - self.assertEqual( - f"{alias.module}.{alias.name}", target) - - def assertDef(self, index, fqname, name, typ): - self.assertIn(fqname, index.defs) - d = index.defs[fqname] - self.assertEqual(d.name, name) - self.assertEqual(d.typ, typ) - - def assertDefLocs(self, index, fqname, locs): - self.assertIn(fqname, index.locs) - deflocs = index.locs[fqname] - self.assertCountEqual([x.location for x in deflocs], locs) - - -class IndexerTest(test_base.BaseTest, IndexerTestMixin): - """Tests for the indexer.""" - - def test_param_reuse(self): - ix = self.index_code(""" - def f(x): - x = 1 # reuse param variable - """.lstrip("\n")) - self.assertDef(ix, "module.f", "f", "FunctionDef") - self.assertDef(ix, "module.f.x", "x", "Param") - self.assertDefLocs(ix, "module.f", [(1, 0)]) - self.assertDefLocs(ix, "module.f.x", [(1, 6), (2, 2)]) - - def test_resolved_imports(self): - # We need all imports to be valid for pytype - code = """ - import f - import x.y - import a.b as c - from a import b - from p import q as r - from u.v import X - from u.v import X as Y - - fx = f.X() - cx = c.X() - bx = b.X() - rx = r.X() - yx = x.y.X() - uvx = X() - X.__name__ - uvy = Y() - Y.__name__ - """ - stub = "class X: pass" - with test_utils.Tempdir() as d: - d.create_file("t.py", code) - d.create_file("f.pyi", stub) - d.create_file(file_utils.replace_separator("x/y.pyi"), stub) - d.create_file(file_utils.replace_separator("a/b.pyi"), stub) - d.create_file(file_utils.replace_separator("p/q.pyi"), stub) - d.create_file(file_utils.replace_separator("u/v.pyi"), stub) - options = config.Options.create(d["t.py"]) - options.tweak(pythonpath=[d.path], version=self.python_version) - ix = indexer.process_file(options) - self.assertDef(ix, "module.c", "c", "Import") - self.assertDef(ix, "module.r", "r", "ImportFrom") - self.assertEqual(ix.modules["module.f"], "f") - self.assertEqual(ix.modules["module.x.y"], "x.y") - self.assertEqual(ix.modules["module.b"], "a.b") - self.assertEqual(ix.modules["module.c"], "a.b") - self.assertEqual(ix.modules["module.r"], "p.q") - self.assertAlias(ix, "module.c", "a.b.<__FILE__>") - self.assertAlias(ix, "module.r", "p.q.<__FILE__>") - self.assertAlias(ix, "module.Y", "u.v.X") - - # Collect all the references from the kythe graph. - kg = kythe.generate_graph(ix, kythe_args=None) - kythe_index = [json.loads(x) for x in output.json_kythe_graph(kg)] - refs = [x for x in kythe_index - if x.get("edge_kind", "").startswith("/kythe/edge/ref")] - - # Extract the span of text and the target symbol for each reference. - src = ix.source.text - out = [] - for r in refs: - pos = r["source"]["signature"] - start, end = pos[1:].split(":") - start, end = int(start), int(end) - text = src[start:end] - out.append((text, r["target"]["signature"], r["target"]["path"])) - - expected = { - # Aliased imports as declarations in the source file - ("c", "module.c", "t.py"), - ("Y", "module.Y", "t.py"), - # Class X in remote files - ("X", "module.X", "f.py"), - ("X", "module.X", "a/b.py"), - ("X", "module.X", "x/y.py"), - ("X", "module.X", "p/q.py"), - # Imports as references to remote files - ("r", "module.r", "t.py"), - ("b", ":module:", "a/b.py"), - ("a.b", ":module:", "a/b.py"), - ("f", ":module:", "f.py"), - ("q", ":module:", "p/q.py"), - ("x.y", ":module:", "x/y.py"), - ("X", "module.X", "u/v.py"), - ("__name__", "module.X.__name__", "u/v.py"), - # x.y as references to remote files - ("x", ":module:", "x/__init__.py"), - ("y", ":module:", "x/y.py"), - # calls - ("X()", "module.X", "p/q.py"), - ("X()", "module.X", "f.py"), - ("X()", "module.X", "a/b.py"), - ("X()", "module.X", "x/y.py"), - ("X()", "module.X", "u/v.py"), - } - - f = file_utils.replace_separator - expected = {(x[0], x[1], f(x[2])) for x in expected} - - # Resolve filepaths within the tempdir. - expected = [(ref, target, d[path]) for (ref, target, path) in expected] - self.assertEqual(set(out), set(expected)) - - def test_source_text(self): - # Don't try to read from the filesystem if we supply source_text - code = textwrap.dedent(""" - def f(x): - return 42 - """) - options = config.Options.create( - file_utils.replace_separator("/path/to/nonexistent/file.py")) - options.tweak(version=self.python_version) - ix = indexer.process_file(options, source_text=code) - self.assertDef(ix, "module.f", "f", "FunctionDef") - - def test_kythe_args(self): - code = textwrap.dedent(""" - def f(x): - return 42 - """) - kythe_index = self.generate_kythe(code) - k = kythe_index[0]["source"] - self.assertEqual(k["corpus"], "corpus") - self.assertEqual(k["root"], "root") - self.assertEqual(k["path"], "path") - - def test_kythe_file_node(self): - code = textwrap.dedent(""" - def f(x): - return 42 - """) - kythe_index = self.generate_kythe(code) - # File nodes should have signature and language empty - file_nodes = kythe_index[0:2] - for node in file_nodes: - self.assertEqual(node["source"]["signature"], "") - self.assertEqual(node["source"]["language"], "") - - # Other nodes should have language="python" - node = kythe_index[3] - self.assertEqual(node["source"]["language"], "python") - - def test_kythe_skip_stdlib(self): - code = textwrap.dedent(""" - import os - """) - kythe_index = self.generate_kythe(code) - imp = [x for x in kythe_index - if x.get("edge_kind") == "/kythe/edge/ref/imports"] - self.assertEqual(len(imp), 1) - kythe_index = self.generate_kythe(code, skip_stdlib=True) - imp = [x for x in kythe_index - if x.get("edge_kind") == "/kythe/edge/ref/imports"] - self.assertFalse(imp) - - def test_multiline_attr(self): - # Test that lookahead doesn't crash. - self.index_code(textwrap.dedent(""" - x = "" - def f(): - return (x. - upper()) - """)) - - def test_literal_attr(self): - # Test that creating a ref id from a literal doesn't crash. - self.index_code(textwrap.dedent(""" - x = {1: 2}.items() - y = [1, 2].reverse() - """)) - - def test_def_types(self): - # Basic sanity test of definition data - ix = self.index_code(""" - def f(): - x = 42 - return x - """) - - def assert_data_type(fqname, cls): - self.assertIn(fqname, ix.defs) - d = ix.defs[fqname] - self.assertEqual(len(d.data), 1) - pytype_cls = d.data[0][0] - self.assertIsInstance(pytype_cls, cls) - - assert_data_type("module.f", abstract.InterpreterFunction) - assert_data_type("module.f.x", abstract.Instance) - - def test_make_serializable(self): - ix = self.index_code(""" - def f(): - x = 42 - y = x - return y - """) - for d in ix.defs.values(): - self.assertIsNotNone(d.data) - for r in ix.refs: - self.assertIsNotNone(r.data) - - ix.make_serializable() - - for d in ix.defs.values(): - self.assertIsNone(d.data) - for r in ix.refs: - self.assertIsNone(r.data) - - def test_docstring(self): - ix = self.index_code(""" - def f(): - '''Multiline docstring - - foo - bar - ''' - """) - d = ix.defs["module.f"] - self.assertEqual(d.doc.text, "Multiline docstring\n\nfoo\n bar") - self.assertGreater(d.doc.length, len(d.doc.text)) - - -class IndexerTestPy3(test_base.BaseTest, IndexerTestMixin): - - def test_type_annotations(self): - ix = self.index_code(""" - def f(x: int) -> int: - return x - """.lstrip("\n")) - self.assertDef(ix, "module.f", "f", "FunctionDef") - self.assertDef(ix, "module.f.x", "x", "Param") - self.assertDefLocs(ix, "module.f", [(1, 0)]) - self.assertDefLocs(ix, "module.f.x", [(1, 6)]) - - -class VmTraceTest(test_base.BaseTest): - - def test_repr(self): - trace = indexer.VmTrace("LOAD_NAME", "x", (["t"],)) - print(repr(trace)) # smoke test - - -if __name__ == "__main__": - test_base.main() diff --git a/pytype/tools/xref/kythe.py b/pytype/tools/xref/kythe.py deleted file mode 100644 index 1a190944c..000000000 --- a/pytype/tools/xref/kythe.py +++ /dev/null @@ -1,324 +0,0 @@ -"""Kythe graph structure.""" - -import base64 -import collections -import dataclasses - -from pytype.tools.xref import utils as xref_utils -from pytype.tools.xref import indexer - -FILE_ANCHOR_SIGNATURE = ":module:" - - -@dataclasses.dataclass(frozen=True) -class Args: - root: str - corpus: str - path: str - skip_stdlib: bool = False - - -# Kythe nodes - - -@dataclasses.dataclass(frozen=True) -class VName: - signature: str - path: str - language: str - root: str - corpus: str - - -@dataclasses.dataclass(frozen=True) -class Fact: - source: VName - fact_name: str - fact_value: str - - -@dataclasses.dataclass(frozen=True) -class Edge: - source: VName - edge_kind: str - target: VName - fact_name: str - - -class Kythe: - """Store a list of kythe graph entries.""" - - def __init__(self, source, args=None): - if args: - self.root = args.root - self.corpus = args.corpus - self.path = args.path or source.filename - self.skip_stdlib = args.skip_stdlib - else: - self.root = "" - self.corpus = "" - self.path = source.filename - self.skip_stdlib = False - self.entries = [] - self._seen_entries = set() - self.file_vname = self._add_file(source.text) - self._add_file_anchor() - - def _encode(self, value): - """Encode fact values as base64.""" - value_bytes = bytes(value, "utf-8") - encoded_bytes = base64.b64encode(value_bytes) - return encoded_bytes.decode("utf-8") - - def _add_file(self, file_contents): - # File vnames are special-cased to have an empty signature and lang. - vname = VName( - signature="", language="", path=self.path, root=self.root, - corpus=self.corpus) - self.add_fact(vname, "node/kind", "file") - self.add_fact(vname, "text", file_contents) - return vname - - def _add_file_anchor(self): - # Add a special anchor for the first byte of a file, so we can link to it. - anchor_vname = self.add_anchor(0, 0) - mod_vname = self.vname(FILE_ANCHOR_SIGNATURE) - self.add_fact(mod_vname, "node/kind", "package") - self.add_edge(anchor_vname, "defines/implicit", mod_vname) - self.add_edge(self.file_vname, "childof", mod_vname) - - def _add_entry(self, entry): - """Make sure we don't have duplicate entries.""" - if entry in self._seen_entries: - return - self._seen_entries.add(entry) - self.entries.append(entry) - - def vname(self, signature, filepath=None, root=None): - return VName( - signature=signature, - path=filepath or self.path, - language="python", - root=root or self.root, - corpus=self.corpus) - - def stdlib_vname(self, signature, filepath=None): - return VName( - signature=signature, - path=filepath or self.path, - language="python", - root=self.root, - corpus="pystdlib") - - def anchor_vname(self, start, end): - signature = "@%d:%d" % (start, end) - return self.vname(signature) - - def fact(self, source, fact_name, fact_value): - fact_name = "/kythe/" + fact_name - fact_value = self._encode(fact_value) - return Fact(source, fact_name, fact_value) - - def edge(self, source, edge_name, target): - edge_kind = "/kythe/edge/" + edge_name - return Edge(source, edge_kind, target, "/") - - def add_fact(self, source, fact_name, fact_value): - fact = self.fact(source, fact_name, fact_value) - self._add_entry(fact) - return fact - - def add_edge(self, source, edge_name, target): - edge = self.edge(source, edge_name, target) - self._add_entry(edge) - return edge - - def add_anchor(self, start, end): - vname = self.anchor_vname(start, end) - self.add_fact(vname, "node/kind", "anchor") - self.add_fact(vname, "loc/start", str(start)) - self.add_fact(vname, "loc/end", str(end)) - return vname - - -# ---------------------------------------------------------------- -# Generate kythe facts from an indexer.Indexer - - -def _process_deflocs(kythe: Kythe, index: indexer.Indexer): - """Generate kythe edges for definitions.""" - - for def_id in index.locs: - defn = index.defs[def_id] - for defloc in index.locs[def_id]: - defn = index.defs[defloc.def_id] - defn_vname = kythe.vname(defn.to_signature()) - start, end = index.get_def_offsets(defloc) - anchor_vname = kythe.add_anchor(start, end) - node_kind = defn.node_kind() - if node_kind == "class": - kythe.add_fact( - source=defn_vname, fact_name="node/kind", fact_value="record") - kythe.add_fact( - source=defn_vname, fact_name="subkind", fact_value=node_kind) - else: - kythe.add_fact( - source=defn_vname, fact_name="node/kind", fact_value=node_kind) - if defn.subkind() is not None: - kythe.add_fact( - source=defn_vname, fact_name="subkind", fact_value=defn.subkind()) - kythe.add_edge( - source=anchor_vname, target=defn_vname, edge_name="defines/binding") - - try: - alias = index.aliases[defn.id] - except KeyError: - pass - else: - alias_vname = _make_defn_vname(kythe, index, alias) - if alias_vname: - kythe.add_edge( - source=defn_vname, target=alias_vname, edge_name="aliases") - - # Emit a docstring if we have one. - doc = defn.doc - if doc: - doc_vname = kythe.vname(defn.doc_signature()) - start, end = index.get_doc_offsets(defn.doc) - anchor_vname = kythe.add_anchor(start, end) - kythe.add_fact( - source=doc_vname, fact_name="node/kind", fact_value="doc") - kythe.add_fact(source=doc_vname, fact_name="text", fact_value=doc.text) - kythe.add_edge( - source=anchor_vname, target=doc_vname, edge_name="defines") - kythe.add_edge( - source=doc_vname, target=defn_vname, edge_name="documents") - - -def _process_params(kythe, index): - """Generate kythe edges for function parameters.""" - - for fp in index.function_params: - fn_def = index.defs[fp.def_id] - param = index.defs[fp.param_id] - kythe.add_edge( - source=kythe.vname(fn_def.to_signature()), - edge_name="param.%d" % fp.position, - target=kythe.vname(param.to_signature())) - - -def _make_defn_vname(kythe, index, defn): - """Convert a definition into a kythe vname.""" - - if isinstance(defn, indexer.Remote): - remote = defn.module - if remote in index.resolved_modules: - is_generated = "generated" in index.resolved_modules[remote].metadata - if remote in index.imports: - # The canonical path from the imports_map is the best bet for - # module->filepath translation. - path = index.imports[remote] - else: - # Fallback to the filepath of the stub file, though this is not always - # accurate due to overrides. - path = index.resolved_modules[remote].filename - path = xref_utils.get_module_filepath(path) - if defn.name == indexer.IMPORT_FILE_MARKER: - sig = FILE_ANCHOR_SIGNATURE - else: - sig = "module." + defn.name - if path.startswith("pytd:"): - if kythe.skip_stdlib: - # Skip builtin and stdlib imports since we don't have a filepath. - # TODO(mdemello): Link to the typeshed definition - return None - return kythe.stdlib_vname( - sig, "pytd:" + index.resolved_modules[remote].module_name) - elif is_generated: - return kythe.vname(sig, path, root="root/genfiles") - else: - return kythe.vname(sig, path) - else: - # Don't generate vnames for unresolved modules. - return None - else: - return kythe.vname(defn.to_signature()) - - -def _process_links(kythe: Kythe, index: indexer.Indexer): - """Generate kythe edges for references.""" - - for ref, defn in index.links: - supported_types = (indexer.Definition, indexer.Remote, indexer.Module) - if not isinstance(defn, supported_types): - # TODO(mdemello): Fixes needed for chained method calls. - continue - start, end = index.get_ref_bounds(ref) - vname = kythe.add_anchor(start, end) - target = _make_defn_vname(kythe, index, defn) - if target is None: - continue - edge_name = "ref" - if ref.typ == "Import" or ref.typ == "ImportFrom": - edge_name = "ref/imports" - kythe.add_edge(source=vname, target=target, edge_name=edge_name) - - -def _process_childof(kythe: Kythe, index: indexer.Indexer): - """Generate kythe edges for childof relationships.""" - - for child, parent in index.childof: - source = _make_defn_vname(kythe, index, child) - target = _make_defn_vname(kythe, index, parent) - kythe.add_edge(source=source, target=target, edge_name="childof") - - -def _process_calls(kythe, index): - """Generate kythe edges for function calls.""" - - # Checks if a function call corresponds to a resolved reference, and generates - # a ref/call to that reference's source definition if so. - - link_map = collections.defaultdict(list) - for ref, defn in index.links: - link_map[ref.location].append((ref, defn)) - - for call in index.calls: - call_links = link_map[call.location] - call_ref = None - call_defn = None - for ref, d in call_links: - if ref.name == call.name: - call_ref = ref - call_defn = d - break - if call_defn: - target = _make_defn_vname(kythe, index, call_defn) - if target: - assert call_ref - start, _ = index.get_ref_bounds(call_ref) - end = index.source.get_offset(call.end_location) - anchor_vname = kythe.add_anchor(start, end) - kythe.add_edge(source=anchor_vname, target=target, edge_name="ref/call") - # The call is a child of the enclosing function/class (this lets us - # generate call graphs). - if ref.scope != "module": # pytype: disable=name-error - parent_defn = index.defs.get(call_ref.scope) - if parent_defn: - # TODO(mdemello): log the 'else' case; it should never happen. - kythe.add_edge( - source=anchor_vname, - target=kythe.vname(parent_defn.to_signature()), - edge_name="childof") - else: - assert False, ref # pytype: disable=name-error - - -def generate_graph(index, kythe_args): - kythe = Kythe(index.source, kythe_args) - _process_deflocs(kythe, index) - _process_params(kythe, index) - _process_links(kythe, index) - _process_childof(kythe, index) - _process_calls(kythe, index) - return kythe diff --git a/pytype/tools/xref/main.py b/pytype/tools/xref/main.py deleted file mode 100644 index 220c8ec6c..000000000 --- a/pytype/tools/xref/main.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Generate cross references from a project.""" - -import signal -import sys -import traceback - -from pytype import utils - -from pytype.tools.xref import debug -from pytype.tools.xref import indexer -from pytype.tools.xref import kythe -from pytype.tools.xref import output -from pytype.tools.xref import parse_args - - -def main(): - try: - args, kythe_args, options = parse_args.parse_args(sys.argv[1:]) - except utils.UsageError as e: - print(str(e), file=sys.stderr) - sys.exit(1) - - if options.timeout is not None and sys.platform != "win32": - signal.alarm(options.timeout) - - try: - ix = indexer.process_file(options, generate_callgraphs=True) - except indexer.PytypeError as e: - print(e.args[0], file=sys.stderr) - if args.debug: - traceback.print_exc() - else: - print("Run with --debug to see a traceback.") - sys.exit(1) - - if args.debug: - debug.show_index(ix) - else: - kythe_graph = kythe.generate_graph(ix, kythe_args) - output.output_kythe_graph(kythe_graph) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/pytype/tools/xref/node_utils.py b/pytype/tools/xref/node_utils.py deleted file mode 100644 index 28c08ecb9..000000000 --- a/pytype/tools/xref/node_utils.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Utility functions for ast nodes.""" - - -def typename(node): - return node.__class__.__name__ - - -def get_name(node, ast): - """Nodes have different name attributes.""" - - if isinstance(node, ast.Attribute): - return get_name(node.value, ast) + "." + node.attr - elif isinstance(node, ast.arg): - return node.arg - elif isinstance(node, str): - return node - elif hasattr(node, "name"): - return node.name - elif hasattr(node, "id"): - return node.id - else: - return "[" + typename(node) + "]" diff --git a/pytype/tools/xref/output.py b/pytype/tools/xref/output.py deleted file mode 100644 index a11ae3fb9..000000000 --- a/pytype/tools/xref/output.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Output utilities for xref.""" - -import dataclasses -import json - - -def unpack(obj): - """Recursively expand namedtuples into dicts.""" - - if hasattr(obj, "_asdict"): - return {k: unpack(v) for k, v in obj._asdict().items()} - elif isinstance(obj, dict): - return {k: unpack(v) for k, v in obj.items()} - elif isinstance(obj, list): - return [unpack(v) for v in obj] - elif isinstance(obj, tuple): - return tuple(unpack(v) for v in obj) - else: - return obj - - -def json_kythe_graph(kythe_graph): - """Generate kythe entries.""" - - for x in kythe_graph.entries: - yield json.dumps(dataclasses.asdict(x)) - - -def output_kythe_graph(kythe_graph): - for x in json_kythe_graph(kythe_graph): - print(x) diff --git a/pytype/tools/xref/parse_args.py b/pytype/tools/xref/parse_args.py deleted file mode 100644 index a67ed7625..000000000 --- a/pytype/tools/xref/parse_args.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Parse command line arguments for xref.""" - -import argparse - -from pytype import config as pytype_config -from pytype import datatypes -from pytype.tools import arg_parser -from pytype.tools.xref import kythe - - -class XrefParser(arg_parser.Parser): - """Subclass the tool parser to retain the raw input field.""" - - def process(self, tool_args, pytype_args): - # Needed for the debug indexer - tool_args.raw_input = pytype_args.input[0] - - -def make_parser(): - """Make parser for command line args. - - Returns: - A Parser object. - """ - - def add_kythe_field(parser, field): - parser.add_argument( - "--" + field, dest=field, type=str, action="store", default="", - help="Part of kythe's file-level vname proto.") - - parser = argparse.ArgumentParser(usage="%(prog)s [options] input") - add_kythe_field(parser, "kythe_corpus") - add_kythe_field(parser, "kythe_root") - add_kythe_field(parser, "kythe_path") - # For the debug indexer - parser.add_argument("--show-types", action="store_true", - dest="show_types", default=None, - help="Display inferred types.") - parser.add_argument("--show-kythe", action="store_true", - dest="show_kythe", default=None, - help="Display kythe facts.") - parser.add_argument("--show-spans", action="store_true", - dest="show_spans", default=None, - help="Display kythe spans.") - # Don't index builtins and stdlib. - parser.add_argument("--skip-stdlib", action="store_true", - dest="skip_stdlib", default=None, - help="Display inferred types.") - # Add options from pytype-single. - wrapper = datatypes.ParserWrapper(parser) - pytype_config.add_basic_options(wrapper) - with wrapper.add_only(["--imports_info", "--debug"]): - pytype_config.add_infrastructure_options(wrapper) - pytype_config.add_debug_options(wrapper) - wrapper.add_argument("input", metavar="input", nargs=1, - help="A .py file to index") - return XrefParser(parser, pytype_single_args=wrapper.actions) - - -def parse_args(argv): - """Parse command line args. - - Arguments: - argv: Raw command line args, typically sys.argv[1:] - - Returns: - A tuple of ( - parsed_args: argparse.Namespace, - kythe_args: kythe.Args, - pytype_options: pytype.config.Options) - """ - - parser = make_parser() - args = parser.parse_args(argv) - t = args.tool_args - kythe_args = kythe.Args( - corpus=t.kythe_corpus, root=t.kythe_root, path=t.kythe_path, - skip_stdlib=t.skip_stdlib - ) - return (args.all_args, kythe_args, args.pytype_opts) diff --git a/pytype/tools/xref/parse_args_test.py b/pytype/tools/xref/parse_args_test.py deleted file mode 100644 index 43d4257da..000000000 --- a/pytype/tools/xref/parse_args_test.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Tests for parse_args.py.""" - -from pytype.tests import test_utils -from pytype.tools.xref import parse_args -import unittest - - -class TestParseArgs(unittest.TestCase): - """Test parse_args.parse_args.""" - - def test_parse_filename(self): - _, _, pytype_opts = parse_args.parse_args(["a.py"]) - self.assertEqual(pytype_opts.input, "a.py") - - def test_parse_no_filename(self): - with self.assertRaises(SystemExit): - parse_args.parse_args([]) - - def test_kythe_args(self): - _, kythe_args, _ = parse_args.parse_args( - ["a.py", - "--kythe_corpus", "foo", - "--kythe_root", "bar", - "--kythe_path", "baz"]) - self.assertEqual(kythe_args.corpus, "foo") - self.assertEqual(kythe_args.root, "bar") - self.assertEqual(kythe_args.path, "baz") - - def test_imports_info(self): - # The code reads and validates an import map within pytype's setup, so we - # need to provide a syntactically valid one as a file. - with test_utils.Tempdir() as d: - pyi_file = d.create_file("baz.pyi") - imports_info = d.create_file("foo", f"bar {pyi_file}") - _, _, opts = parse_args.parse_args( - ["a.py", "--imports_info", imports_info]) - self.assertEqual(opts.imports_map.items, {"bar": pyi_file}) - self.assertTrue(opts.use_pickled_files) - - -if __name__ == "__main__": - unittest.main() diff --git a/pytype/tools/xref/scripts/README b/pytype/tools/xref/scripts/README deleted file mode 100644 index fc83f079f..000000000 --- a/pytype/tools/xref/scripts/README +++ /dev/null @@ -1,5 +0,0 @@ -You can find prebuilt binaries at https://github.com/kythe/kythe/releases. -These scripts assume that they are installed to $HOME/kythe. - -If you build the tools yourself or install them to a different location, -make sure to pass the correct public_resources directory to http_server. diff --git a/pytype/tools/xref/scripts/kythe-add-index.sh b/pytype/tools/xref/scripts/kythe-add-index.sh deleted file mode 100755 index a2abb4654..000000000 --- a/pytype/tools/xref/scripts/kythe-add-index.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -e -set -o pipefail - -# Add a file's graph entries to the kythe index. -# Usage: -# xref file | kythe-add-index.sh - -# The easiest way to install kythe is as user in your home directory. -KYTHE_HOME="$HOME/kythe" -KYTHE_TOOLS="$KYTHE_HOME/tools" - -# Read JSON entries from standard in to a graphstore. -$KYTHE_TOOLS/entrystream --read_format=json \ - | $KYTHE_TOOLS/write_entries -graphstore graphstore diff --git a/pytype/tools/xref/scripts/kythe-browse.sh b/pytype/tools/xref/scripts/kythe-browse.sh deleted file mode 100755 index a190faac1..000000000 --- a/pytype/tools/xref/scripts/kythe-browse.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -e -set -o pipefail - -# Launch the kythe browser - -BROWSE_PORT="${BROWSE_PORT:-8080}" - -# The easiest way to install kythe is as user in your home directory. -KYTHE_HOME="$HOME/kythe" -KYTHE_TOOLS="$KYTHE_HOME/tools" - -# Convert the graphstore to serving tables. -$KYTHE_TOOLS/write_tables -graphstore graphstore -out=tables - -# Host the browser UI. -$KYTHE_TOOLS/http_server -serving_table tables \ - -public_resources="${KYTHE_HOME}/web/ui" \ - -listen="localhost:${BROWSE_PORT}" diff --git a/pytype/tools/xref/scripts/kythe-clean.sh b/pytype/tools/xref/scripts/kythe-clean.sh deleted file mode 100755 index 3c9929111..000000000 --- a/pytype/tools/xref/scripts/kythe-clean.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# Clean out the kythe index - -rm -f -- graphstore/* tables/* -mkdir -p graphstore tables diff --git a/pytype/tools/xref/scripts/kythe-verifier.sh b/pytype/tools/xref/scripts/kythe-verifier.sh deleted file mode 100755 index f1764ffe8..000000000 --- a/pytype/tools/xref/scripts/kythe-verifier.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -e -set -o pipefail - -# Run the kythe verifier -# Usage: -# xref | kythe-verifier.sh - -# The easiest way to install kythe is as user in your home directory. -KYTHE_HOME="$HOME/kythe" -KYTHE_TOOLS="$KYTHE_HOME/tools" - -# Read JSON entries from standard in and pass them to the verifier. -$KYTHE_TOOLS/entrystream --read_format=json \ - | $KYTHE_TOOLS/verifier --goal_prefix="#-" "$@" diff --git a/pytype/tools/xref/testdata/CMakeLists.txt b/pytype/tools/xref/testdata/CMakeLists.txt deleted file mode 100644 index f47351aaf..000000000 --- a/pytype/tools/xref/testdata/CMakeLists.txt +++ /dev/null @@ -1,127 +0,0 @@ -add_package() - -py_library( - NAME - multiline - SRCS - multiline.py -) - -py_library( - NAME - function_call - SRCS - function_call.py -) - -py_library( - NAME - nested_class - SRCS - nested_class.py -) - -py_library( - NAME - subclass - SRCS - subclass.py -) - -py_library( - NAME - attr - SRCS - attr.py -) - -py_library( - NAME - imports - SRCS - imports.py -) - -py_library( - NAME - class_def - SRCS - class_def.py -) - -py_library( - NAME - scope - SRCS - scope.py -) - -py_library( - NAME - docstrings - SRCS - docstrings.py -) - -py_library( - NAME - method_call - SRCS - method_call.py -) - -py_library( - NAME - nested_definitions - SRCS - nested_definitions.py -) - -py_library( - NAME - variables - SRCS - variables.py -) - -py_library( - NAME - builtins - SRCS - builtins.py -) - -py_library( - NAME - module - SRCS - module.py -) - -py_library( - NAME - stdlib - SRCS - stdlib.py -) - -py_library( - NAME - minimal - SRCS - minimal.py -) - -py_library( - NAME - call - SRCS - call.py -) - -py_library( - NAME - params - SRCS - params.py -) diff --git a/pytype/tools/xref/testdata/attr.py b/pytype/tools/xref/testdata/attr.py deleted file mode 100644 index 2ab4ac4eb..000000000 --- a/pytype/tools/xref/testdata/attr.py +++ /dev/null @@ -1,70 +0,0 @@ -# pylint: skip-file - -#- @A defines/binding ClassA -#- ClassA.node/kind record -#- ClassA.subkind class -class A: - #- @__init__ defines/binding FnInit - #- @self defines/binding ArgSelf - #- FnInit.node/kind function - #- FnInit param.0 ArgSelf - def __init__(self): - #- @self ref ArgSelf - #- @foo defines/binding AttrFoo - self.foo = [] - - def f(self, x): - #- @foo ref AttrFoo - self.foo[x] = 10 - - -## The attr can be initialised somewhere other than __init__ - -#- @B defines/binding ClassB -#- ClassB.node/kind record -#- ClassB.subkind class -class B: - def f(self, x): - #- @bar ref AttrBar - self.bar[x] - - #- @init_bar defines/binding FnInitBar - #- @self defines/binding ArgBSelf - #- FnInitBar.node/kind function - #- FnInitBar param.0 ArgBSelf - def init_bar(self): - #- @self ref ArgBSelf - #- @bar defines/binding AttrBar - self.bar = [] - return self - - ## Attribute accesses could span several lines - def baz(self): - (self. - #- @init_bar ref FnInitBar - init_bar() - #- @bar ref AttrBar - .bar) - - -#- @C defines/binding ClassC -#- ClassC.node/kind record -#- ClassC.subkind class -class C: - #- @x defines/binding AttrX - x: int - - def f(self): - #- @x ref AttrX - return self.x - - def g(notself): - #- @y defines/binding _ - notself.y = 10 - - -#- @x ref AttrX -y = C().x -c = C() -#- @x ref AttrX -c.x = 1 diff --git a/pytype/tools/xref/testdata/builtins.py b/pytype/tools/xref/testdata/builtins.py deleted file mode 100644 index f80e3f2b8..000000000 --- a/pytype/tools/xref/testdata/builtins.py +++ /dev/null @@ -1,7 +0,0 @@ -# pylint: skip-file - -a = "hello" -#- @split ref vname("module.str.split", _, _, "pytd:builtins", _) -b = a.split('.') -#- @reverse ref vname("module.list.reverse", _, _, "pytd:builtins", _) -c = b.reverse() diff --git a/pytype/tools/xref/testdata/call.py b/pytype/tools/xref/testdata/call.py deleted file mode 100644 index a1fa59459..000000000 --- a/pytype/tools/xref/testdata/call.py +++ /dev/null @@ -1,18 +0,0 @@ -# pylint: skip-file - -#- @foo defines/binding FnFoo -#- FnFoo.node/kind function -def foo(x): - pass - -#- @foo ref FnFoo -#- @"foo(123)" ref/call FnFoo -x = foo(123) - -#- @bar defines/binding FnBar -#- FnBar.node/kind function -def bar(x): - #- @foo ref FnFoo - #- @"foo(123)" ref/call FnFoo - #- @"foo(123)" childof FnBar - x = foo(123) diff --git a/pytype/tools/xref/testdata/class_def.py b/pytype/tools/xref/testdata/class_def.py deleted file mode 100644 index 70736c77c..000000000 --- a/pytype/tools/xref/testdata/class_def.py +++ /dev/null @@ -1,63 +0,0 @@ -# pylint: skip-file - -import collections - - -#- @A defines/binding ClassA -#- ClassA.node/kind record -#- ClassA.subkind class -class A: - pass - - -#- @B defines/binding ClassB -#- ClassB.node/kind record -#- ClassB.subkind class -class B: - pass - - -#- @D defines/binding ClassD -#- @A ref ClassA -#- ClassD.node/kind record -#- ClassD.subkind class -class D(A): - pass - - -#- @Foo defines/binding ClassFoo -#- @A ref ClassA -#- @B ref ClassB -#- ClassFoo.node/kind record -#- ClassFoo.subkind class -class Foo(A, B): - pass - - -#- @Bar defines/binding ClassBar -#- ClassBar.node/kind record -#- ClassBar.subkind class -class Bar( - #- @A ref ClassA - A, - #- @B ref ClassB - B): - pass - - -#- @Baz defines/binding ClassBaz -#- ClassBaz.node/kind record -#- ClassBaz.subkind class -class Baz(collections.namedtuple('Foo', ['bar', 'baz', 'quux'])): - pass - - -#- @Quux defines/binding ClassQuux -#- ClassQuux.node/kind record -#- ClassQuux.subkind class -class Quux: - pass - - -def f(): - global Quux diff --git a/pytype/tools/xref/testdata/docstrings.py b/pytype/tools/xref/testdata/docstrings.py deleted file mode 100644 index 1945b66ec..000000000 --- a/pytype/tools/xref/testdata/docstrings.py +++ /dev/null @@ -1,18 +0,0 @@ -# pylint: skip-file - -#- @A defines/binding ClassA -#- ClassA.node/kind record -#- ClassA.subkind class -class A: - #- ClassDoc documents ClassA - #- ClassDoc.node/kind doc - #- ClassDoc.text "Class docstring" - """Class docstring""" - #- @foo defines/binding FnFoo - #- FnFoo.node/kind function - def foo(self, x): - #- MethodDoc documents FnFoo - #- MethodDoc.node/kind doc - #- MethodDoc.text "Method docstring" - """Method docstring""" - pass diff --git a/pytype/tools/xref/testdata/function_call.py b/pytype/tools/xref/testdata/function_call.py deleted file mode 100644 index ca61b286e..000000000 --- a/pytype/tools/xref/testdata/function_call.py +++ /dev/null @@ -1,37 +0,0 @@ -# pylint: skip-file - -#- @test defines/binding FnTest -#- @x defines/binding ArgX -#- FnTest.node/kind function -#- FnTest param.0 ArgX -def test(x): - #- @x ref ArgX - return x - - -#- @foo defines/binding FnFoo -#- @x defines/binding ArgFooX -#- FnFoo.node/kind function -#- FnFoo param.0 ArgFooX -def foo(x): - #- @"test(x)" ref/call FnTest - #- @"test(x)" childof FnFoo - return test(x) - - -#- @y defines/binding VarY -#- @test ref FnTest -#- @"test(2)" ref/call FnTest -#- VarY.node/kind variable -y = test(2) - - -# We don't index this, but it shouldn't crash. -z = (lambda x: x)(1) - - -#- @bar defines/binding VarBar -bar = lambda x: x -# We don't generate ref/call here. -#- @bar ref VarBar -bar(1) diff --git a/pytype/tools/xref/testdata/imports.py b/pytype/tools/xref/testdata/imports.py deleted file mode 100644 index 51992d00a..000000000 --- a/pytype/tools/xref/testdata/imports.py +++ /dev/null @@ -1,32 +0,0 @@ -# pylint: skip-file - -#- @#0os ref/imports ModuleOs -import os -#- @#0os ref/imports ModuleOs -#- @os_alias defines/binding OsAlias -#- OsAlias.node/kind variable -#- OsAlias.subkind import -#- OsAlias aliases ModuleOs -import os as os_alias - -#- @"os.path" ref/imports ModuleOsPath -import os.path -#- @os ref ModuleOs -#- @path ref ModuleOsPath -os.path.exists - -#- @path ref/imports ModuleOsPath -from os import path -#- @path ref ModuleOsPath -path.exists - -#- @name ref OsName -os.name -#- @name ref/imports OsName -from os import name - -#- @name_alias defines/binding OsNameAlias -#- OsNameAlias.node/kind variable -#- OsNameAlias.subkind import -#- OsNameAlias aliases OsName -from os import name as name_alias diff --git a/pytype/tools/xref/testdata/method_call.py b/pytype/tools/xref/testdata/method_call.py deleted file mode 100644 index 205b1e654..000000000 --- a/pytype/tools/xref/testdata/method_call.py +++ /dev/null @@ -1,26 +0,0 @@ -# pylint: skip-file - -#- @A defines/binding ClassA -#- ClassA.node/kind record -#- ClassA.subkind class -class A: - #- @foo defines/binding FnFoo - #- @self defines/binding ArgSelf - #- @x defines/binding ArgX - #- FnFoo.node/kind function - #- FnFoo param.0 ArgSelf - #- FnFoo param.1 ArgX - def foo(self, x): - pass - - -#- @bar defines/binding FnBar -#- FnBar.node/kind function -def bar(): - #- @A ref ClassA - #- @"A()" ref/call ClassA - #- @"A()" childof FnBar - #- @foo ref FnFoo - #- @"foo(10)" ref/call FnFoo - #- @"foo(10)" childof FnBar - A().foo(10) diff --git a/pytype/tools/xref/testdata/minimal.py b/pytype/tools/xref/testdata/minimal.py deleted file mode 100644 index bf25b914c..000000000 --- a/pytype/tools/xref/testdata/minimal.py +++ /dev/null @@ -1,5 +0,0 @@ -# pylint: skip-file -"""Minimal file contents to satisfy existence requirements.""" - -# Minimal goal in order to define at least one goal. -#- _.node/kind file diff --git a/pytype/tools/xref/testdata/module.py b/pytype/tools/xref/testdata/module.py deleted file mode 100644 index 37fa8ae7d..000000000 --- a/pytype/tools/xref/testdata/module.py +++ /dev/null @@ -1,13 +0,0 @@ -# pylint: skip-file -#- File.node/kind file -#- File childof Mod -#- Mod=vname(":module:", _, _, _, _).node/kind package - -# The first byte in the module is tagged as defining the module. -# (Idea copied from the typescript indexer) -#- ModAnchor.node/kind anchor -#- ModAnchor./kythe/loc/start 0 -#- ModAnchor./kythe/loc/end 0 -#- ModAnchor defines/implicit Mod - -X = 42 diff --git a/pytype/tools/xref/testdata/multiline.py b/pytype/tools/xref/testdata/multiline.py deleted file mode 100644 index 614ea60b6..000000000 --- a/pytype/tools/xref/testdata/multiline.py +++ /dev/null @@ -1,38 +0,0 @@ -# pylint: skip-file - -#- @a defines/binding VarA -#- @b defines/binding VarB -#- @c defines/binding VarC -a = b = c = 1 - -# TODO(b/137314416): @a ref VarA does not work -#- @x defines/binding VarX -x = (a + -#- @b ref VarB - b + -#- @c ref VarC - c) - -#- @y defines/binding VarY -#- @a ref VarA -y = list([a, -#- @b ref VarB - b, -#- @c ref VarC - c]) - -#- @x ref VarX -#- @y ref VarY -#- @z defines/binding VarZ -z = [x, y] - -#- @z ref VarZ -z[1] = (1, - 2) - -#- @foo defines/binding Foo -#- Foo.node/kind function -def foo(x, - y, - z): - pass diff --git a/pytype/tools/xref/testdata/nested_class.py b/pytype/tools/xref/testdata/nested_class.py deleted file mode 100644 index e8251a72e..000000000 --- a/pytype/tools/xref/testdata/nested_class.py +++ /dev/null @@ -1,34 +0,0 @@ -# pylint: skip-file - -#- @A defines/binding ClassA -#- ClassA.node/kind record -#- ClassA.subkind class -class A: - #- @B defines/binding ClassB - #- ClassB.node/kind record - #- ClassB.subkind class - class B: - #- @foo defines/binding FnFoo - #- @self defines/binding ArgBSelf - #- FnFoo.node/kind function - #- FnFoo param.0 ArgBSelf - def foo(self): - pass - - #- @bar defines/binding FnBar - #- @self defines/binding ArgASelf - #- FnBar.node/kind function - #- FnBar param.0 ArgASelf - def bar(self): - #- @B ref ClassB - return self.B() - -#- @A ref ClassA -#- @B ref ClassB -#- @foo ref FnFoo -A.B().foo() - -#- @A ref ClassA -#- @bar ref FnBar -#- @foo ref FnFoo -A().bar().foo() diff --git a/pytype/tools/xref/testdata/nested_definitions.py b/pytype/tools/xref/testdata/nested_definitions.py deleted file mode 100644 index 587867379..000000000 --- a/pytype/tools/xref/testdata/nested_definitions.py +++ /dev/null @@ -1,57 +0,0 @@ -# pylint: skip-file - -def f(): - #- @A defines/binding ClassA - #- ClassA.node/kind record - #- ClassA.subkind class - class A: - pass - - -#- @g defines/binding G -def g(): - #- @B defines/binding ClassB - #- ClassB.node/kind record - #- ClassB.subkind class - #- ClassB childof G - class B: - pass - - return B - - -#- @h defines/binding H -def h(base): - #- @C defines/binding ClassC - #- ClassC.node/kind record - #- ClassC.subkind class - #- ClassC childof H - class C(base): - pass - - #- @D defines/binding ClassD - #- ClassD.node/kind record - #- ClassD.subkind class - #- ClassD childof H - class D(C): - pass - - return D() - - -#- @Outer defines/binding Outer -class Outer: - #- @method defines/binding Method - #- Method childof Outer - def method(self): - #- @NestedClass defines/binding NestedClass - #- NestedClass childof Method - class NestedClass: - #- @fn defines/binding Fn - #- Fn childof NestedClass - def fn(): - pass - #- @nested_fn defines/binding NestedFn - #- NestedFn childof Method - def nested_fn(): - pass diff --git a/pytype/tools/xref/testdata/params.py b/pytype/tools/xref/testdata/params.py deleted file mode 100644 index 0593dcc82..000000000 --- a/pytype/tools/xref/testdata/params.py +++ /dev/null @@ -1,32 +0,0 @@ -# pylint: skip-file - -#- @foo defines/binding FnFoo -#- @x defines/binding ArgX -#- @y defines/binding ArgY -#- @z defines/binding ArgZ -#- @a defines/binding ArgA -#- @b defines/binding ArgB -#- FnFoo.node/kind function -#- FnFoo param.0 ArgX -#- FnFoo param.1 ArgY -#- FnFoo param.2 ArgZ -#- FnFoo param.3 ArgA -#- FnFoo param.4 ArgB -def foo(x, y, /, z, *, a, b): - #- @x ref ArgX - return x - - -#- @bar defines/binding FnBar -#- @p defines/binding ArgP -#- @varargs defines/binding ArgVarargs -#- @q defines/binding ArgQ -#- @kwargs defines/binding ArgKwargs -#- FnBar param.0 ArgP -#- FnBar param.1 ArgVarargs -#- FnBar param.2 ArgQ -#- FnBar param.3 ArgKwargs -def bar(p, *varargs, q, **kwargs): - #- @varargs ref ArgVarargs - #- @kwargs ref ArgKwargs - return varargs, kwargs diff --git a/pytype/tools/xref/testdata/scope.py b/pytype/tools/xref/testdata/scope.py deleted file mode 100644 index 402537107..000000000 --- a/pytype/tools/xref/testdata/scope.py +++ /dev/null @@ -1,28 +0,0 @@ -# pylint: skip-file - -#- @foo defines/binding VarFoo -#- VarFoo.node/kind variable -foo = 5 - -def fun(): - #- @bar defines/binding VarBar - #- VarBar.node/kind variable - bar = 10 - - def nested(): - #- @baz defines/binding VarBaz - #- VarBaz.node/kind variable - baz = 10 - #- @foo ref VarFoo - foo - #- @bar ref VarBar - bar - #- @baz ref VarBaz - baz - # Two references to the same variable should both get linked correctly. - #- @foo ref VarFoo - foo - #- @bar ref VarBar - bar - #- @baz ref VarBaz - baz diff --git a/pytype/tools/xref/testdata/stdlib.py b/pytype/tools/xref/testdata/stdlib.py deleted file mode 100644 index 8ff0739cd..000000000 --- a/pytype/tools/xref/testdata/stdlib.py +++ /dev/null @@ -1,7 +0,0 @@ -# pylint: skip-file - -import os - -#- @path ref vname(":module:", "pystdlib", _, "pytd:os.path", "python") -#- @split ref vname("module.split", "pystdlib", _, "pytd:os.path", "python") -os.path.split("/x/y") diff --git a/pytype/tools/xref/testdata/subclass.py b/pytype/tools/xref/testdata/subclass.py deleted file mode 100644 index 76825b745..000000000 --- a/pytype/tools/xref/testdata/subclass.py +++ /dev/null @@ -1,39 +0,0 @@ -# pylint: skip-file - -#- @A defines/binding ClassA -#- ClassA.node/kind record -#- ClassA.subkind class -class A: - #- @foo defines/binding FnFoo - #- FnFoo.node/kind function - def foo(self, x): - return 10 - - @staticmethod - #- @bar defines/binding FnBar - #- FnBar.node/kind function - def bar(): - return 42 - - -class B(A): - pass - - -class C(B): - #- @baz defines/binding FnBaz - #- @self defines/binding ArgSelf - #- FnBaz.node/kind function - #- FnBaz param.0 ArgSelf - def baz(self): - #- @self ref ArgSelf - #- @"foo(10)" ref/call FnFoo - return self.foo(10) - - -#- @foo ref FnFoo -#- @"foo(10)" ref/call FnFoo -x = B().foo(10) -#- @bar ref FnBar -#- @"bar()" ref/call FnBar -y = C.bar() diff --git a/pytype/tools/xref/testdata/variables.py b/pytype/tools/xref/testdata/variables.py deleted file mode 100644 index 84b185e7f..000000000 --- a/pytype/tools/xref/testdata/variables.py +++ /dev/null @@ -1,20 +0,0 @@ -# pylint: skip-file - -class A(): - #- @foo defines/binding FnFoo - def foo(self): - pass - -# Follow a type through a chain of variables - -#- @x defines/binding VarX -#- VarX.node/kind variable -x = A() -#- @y defines/binding VarY -#- VarY.node/kind variable -y = x -#- @z defines/binding VarZ -#- VarZ.node/kind variable -z = y -#- @foo ref FnFoo -z.foo() diff --git a/pytype/tools/xref/utils.py b/pytype/tools/xref/utils.py deleted file mode 100644 index f2d67a9f0..000000000 --- a/pytype/tools/xref/utils.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Utilities for xref.""" - -from pytype import module_utils -from pytype.platform_utils import path_utils - - -def get_module_filepath(module_filename): - """Recover the path to the py file from a module pyi path.""" - - def _clean(path): - """Change extension to .py.""" - prefix, fname = path_utils.split(path) - fname, _ = path_utils.splitext(fname) - path = path_utils.join(prefix, fname + ".py") - return path - - return _clean(module_filename) - - -def process_imports_map(imports_map): - """Generate a map of {module name: canonical relative path}.""" - if not imports_map: - return {} - - # Store maps of the full path and canonical relative path. - mod_to_fp = {} - fp_to_cp = {} - - for path, v in imports_map.items(): - mod = module_utils.path_to_module_name(path) - mod_to_fp[mod] = v - if v not in fp_to_cp or len(path) > len(fp_to_cp[v]): - fp_to_cp[v] = path - - return {mod: fp_to_cp[fp] for mod, fp in mod_to_fp.items()}