fix(gazelle): embed default stdlib list for non-Bazel Go builds (#4197) `gazelle/python/std_modules.go` previously used `//go:embed stdlib_list.txt`, where `stdlib_list.txt` only existed as a Bazel `copy_file` output from `@python_stdlib_list`. Building, listing, or testing `github.com/bazel-contrib/rules_python/gazelle/python` directly with standard Go tooling (`go build`, `go test`, `go list -deps`, or `go install` for standalone Gazelle binaries in pre-commit hooks) failed with `pattern stdlib_list.txt: no matching files found`, and running `bazel run //:gazelle` inside `gazelle/` emitted a missing-file warning on `std_modules.go`. Check in the default standard library module list at `gazelle/python/stdlib_list/default.txt` (verified via `diff_test` against `@python_stdlib_list//:stdlib_list/lists/3.14.txt`), change the Bazel `copy_file` output to `stdlib_list/selected.txt`, and embed `stdlib_list/*.txt` via `embed.FS`. At initialization, `loadStdModules` prefers `stdlib_list/selected.txt` when present (preserving Bazel's `python_version` `select()` behavior) and falls back to `stdlib_list/default.txt` for non-Bazel Go builds. Fixes #3821 Co-authored-by: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
diff --git a/gazelle/python/BUILD.bazel b/gazelle/python/BUILD.bazel index 0b9fd76..d395f7c 100644 --- a/gazelle/python/BUILD.bazel +++ b/gazelle/python/BUILD.bazel
@@ -1,6 +1,8 @@ load("@bazel_gazelle//:def.bzl", "gazelle_binary") load("@bazel_skylib//rules:copy_file.bzl", "copy_file") +load("@bazel_skylib//rules:diff_test.bzl", "diff_test") load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_python//python:defs.bzl", "py_binary") load(":gazelle_test.bzl", "gazelle_test") go_library( @@ -17,15 +19,10 @@ "std_modules.go", "target.go", ], - # NOTE @aignas 2023-12-03: currently gazelle does not support embedding - # generated files, but 3.11.txt is generated by a build rule. - # - # You will get a benign error like when running gazelle locally: - # > 8 gazelle: .../rules_python/gazelle/python/std_modules.go:24:3: pattern 3.11.txt: matched no files - # - # See following for more info: - # https://github.com/bazelbuild/bazel-gazelle/issues/1513 - embedsrcs = ["stdlib_list.txt"], # keep # TODO: use user-defined version? + embedsrcs = [ + "stdlib_list/default.txt", + "stdlib_list/selected.txt", # keep # TODO: use user-defined version? + ], importpath = "github.com/bazel-contrib/rules_python/gazelle/python", visibility = ["//visibility:public"], deps = [ @@ -47,6 +44,8 @@ ], ) +_DEFAULT_STDLIB_LIST = "@python_stdlib_list//:stdlib_list/lists/3.14.txt" + copy_file( name = "stdlib_list", src = select( @@ -59,13 +58,32 @@ "@rules_python//python/config_settings:is_python_3.8": "@python_stdlib_list//:stdlib_list/lists/3.8.txt", "@rules_python//python/config_settings:is_python_3.9": "@python_stdlib_list//:stdlib_list/lists/3.9.txt", # Fall back to the latest stdlib list - "//conditions:default": "@python_stdlib_list//:stdlib_list/lists/3.14.txt", + "//conditions:default": _DEFAULT_STDLIB_LIST, }, ), - out = "stdlib_list.txt", + out = "stdlib_list/selected.txt", allow_symlink = True, ) +diff_test( + name = "stdlib_list_default_test", + failure_message = "Run 'bazel run //python:stdlib_list_default.update' (from gazelle/) to sync stdlib_list/default.txt.", + file1 = "stdlib_list/default.txt", + file2 = _DEFAULT_STDLIB_LIST, +) + +py_binary( + name = "stdlib_list_default.update", + srcs = ["//manifest:copy_to_source.py"], + args = [ + "$(rootpath {})".format(_DEFAULT_STDLIB_LIST), + "python/stdlib_list/default.txt", + ], + data = [_DEFAULT_STDLIB_LIST], + main = "//manifest:copy_to_source.py", + tags = ["manual"], +) + # gazelle:exclude testdata/ # gazelle:exclude extensions.bzl # Exclude test-only Starlark helper from Gazelle to avoid generating an unnecessary bzl_library
diff --git a/gazelle/python/std_modules.go b/gazelle/python/std_modules.go index ecb4f4c..b3f2c59 100644 --- a/gazelle/python/std_modules.go +++ b/gazelle/python/std_modules.go
@@ -16,24 +16,41 @@ import ( "bufio" - _ "embed" - "strings" + "bytes" + "embed" + "io/fs" ) var ( - //go:embed stdlib_list.txt - stdlibList string + //go:embed stdlib_list/*.txt + stdlibFS embed.FS stdModules map[string]struct{} ) func init() { - stdModules = make(map[string]struct{}) - scanner := bufio.NewScanner(strings.NewReader(stdlibList)) - for scanner.Scan() { - stdModules[scanner.Text()] = struct{}{} + var err error + stdModules, err = loadStdModules(stdlibFS) + if err != nil { + panic(err) } } +func loadStdModules(fsys fs.ReadFileFS) (map[string]struct{}, error) { + data, err := fsys.ReadFile("stdlib_list/selected.txt") + if err != nil { + data, err = fsys.ReadFile("stdlib_list/default.txt") + if err != nil { + return nil, err + } + } + modules := make(map[string]struct{}) + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + modules[scanner.Text()] = struct{}{} + } + return modules, scanner.Err() +} + func isStdModule(m Module) bool { _, ok := stdModules[m.Name] return ok
diff --git a/gazelle/python/std_modules_test.go b/gazelle/python/std_modules_test.go index dbcd18c..632a3d7 100644 --- a/gazelle/python/std_modules_test.go +++ b/gazelle/python/std_modules_test.go
@@ -16,6 +16,7 @@ import ( "testing" + "testing/fstest" "github.com/stretchr/testify/assert" ) @@ -25,3 +26,27 @@ assert.True(t, isStdModule(Module{Name: "os.path"})) assert.False(t, isStdModule(Module{Name: "foo"})) } + +func TestLoadStdModules(t *testing.T) { + t.Run("prefers selected.txt when present", func(t *testing.T) { + fsys := fstest.MapFS{ + "stdlib_list/default.txt": &fstest.MapFile{Data: []byte("default_only\n")}, + "stdlib_list/selected.txt": &fstest.MapFile{Data: []byte("selected_only\n")}, + } + modules, err := loadStdModules(fsys) + if assert.NoError(t, err) { + assert.Contains(t, modules, "selected_only") + assert.NotContains(t, modules, "default_only") + } + }) + + t.Run("falls back to default.txt when selected.txt is absent", func(t *testing.T) { + fsys := fstest.MapFS{ + "stdlib_list/default.txt": &fstest.MapFile{Data: []byte("default_only\n")}, + } + modules, err := loadStdModules(fsys) + if assert.NoError(t, err) { + assert.Contains(t, modules, "default_only") + } + }) +}
diff --git a/gazelle/python/stdlib_list/default.txt b/gazelle/python/stdlib_list/default.txt new file mode 100644 index 0000000..183a15e --- /dev/null +++ b/gazelle/python/stdlib_list/default.txt
@@ -0,0 +1,827 @@ +__future__ +__main__ +_abc +_aix_support +_android_support +_apple_support +_ast +_ast_unparse +_asyncio +_bisect +_blake2 +_bz2 +_codecs +_codecs_cn +_codecs_hk +_codecs_iso2022 +_codecs_jp +_codecs_kr +_codecs_tw +_collections +_collections_abc +_colorize +_compat_pickle +_contextvars +_csv +_ctypes +_curses +_curses_panel +_datetime +_dbm +_decimal +_elementtree +_frozen_importlib +_frozen_importlib_external +_functools +_gdbm +_hashlib +_heapq +_hmac +_imp +_interpchannels +_interpqueues +_interpreters +_io +_ios_support +_json +_locale +_lsprof +_lzma +_markupbase +_md5 +_multibytecodec +_multiprocessing +_opcode +_opcode_metadata +_operator +_osx_support +_overlapped +_pickle +_posixshmem +_posixsubprocess +_py_abc +_py_warnings +_pydatetime +_pydecimal +_pyio +_pylong +_pyrepl +_pyrepl.__main__ +_pyrepl._module_completer +_pyrepl._threading_handler +_pyrepl.base_eventqueue +_pyrepl.commands +_pyrepl.completing_reader +_pyrepl.console +_pyrepl.fancy_termios +_pyrepl.historical_reader +_pyrepl.input +_pyrepl.keymap +_pyrepl.main +_pyrepl.pager +_pyrepl.reader +_pyrepl.readline +_pyrepl.simple_interact +_pyrepl.terminfo +_pyrepl.trace +_pyrepl.types +_pyrepl.unix_console +_pyrepl.unix_eventqueue +_pyrepl.utils +_pyrepl.windows_console +_pyrepl.windows_eventqueue +_queue +_random +_remote_debugging +_scproxy +_sha1 +_sha2 +_sha3 +_signal +_sitebuiltins +_socket +_sqlite3 +_sre +_ssl +_stat +_statistics +_string +_strptime +_struct +_suggestions +_symtable +_sysconfig +_thread +_threading_local +_tkinter +_tokenize +_tracemalloc +_types +_typing +_uuid +_warnings +_weakref +_weakrefset +_winapi +_wmi +_zoneinfo +_zstd +abc +annotationlib +antigravity +argparse +array +ast +asyncio +asyncio.__main__ +asyncio.base_events +asyncio.base_futures +asyncio.base_subprocess +asyncio.base_tasks +asyncio.constants +asyncio.coroutines +asyncio.events +asyncio.exceptions +asyncio.format_helpers +asyncio.futures +asyncio.graph +asyncio.locks +asyncio.log +asyncio.mixins +asyncio.proactor_events +asyncio.protocols +asyncio.queues +asyncio.runners +asyncio.selector_events +asyncio.sslproto +asyncio.staggered +asyncio.streams +asyncio.subprocess +asyncio.taskgroups +asyncio.tasks +asyncio.threads +asyncio.timeouts +asyncio.tools +asyncio.transports +asyncio.trsock +asyncio.unix_events +asyncio.windows_events +asyncio.windows_utils +atexit +base64 +bdb +binascii +bisect +builtins +bz2 +cProfile +calendar +cmath +cmd +code +codecs +codeop +collections +colorsys +compileall +compression +compression._common +compression._common._streams +compression.bz2 +compression.gzip +compression.lzma +compression.zlib +compression.zstd +compression.zstd._zstdfile +concurrent +concurrent.futures +concurrent.futures._base +concurrent.futures.interpreter +concurrent.futures.process +concurrent.futures.thread +concurrent.interpreters +concurrent.interpreters._crossinterp +concurrent.interpreters._queues +configparser +contextlib +contextvars +copy +copyreg +csv +ctypes +ctypes._aix +ctypes._endian +ctypes._layout +ctypes.macholib +ctypes.macholib.dyld +ctypes.macholib.dylib +ctypes.macholib.framework +ctypes.util +ctypes.wintypes +curses +curses.ascii +curses.has_key +curses.panel +curses.textpad +dataclasses +datetime +dbm +dbm.dumb +dbm.gnu +dbm.ndbm +dbm.sqlite3 +decimal +difflib +dis +doctest +email +email._encoded_words +email._header_value_parser +email._parseaddr +email._policybase +email.base64mime +email.charset +email.contentmanager +email.encoders +email.errors +email.feedparser +email.generator +email.header +email.headerregistry +email.iterators +email.message +email.mime +email.mime.application +email.mime.audio +email.mime.base +email.mime.image +email.mime.message +email.mime.multipart +email.mime.nonmultipart +email.mime.text +email.parser +email.policy +email.quoprimime +email.utils +encodings +encodings._win_cp_codecs +encodings.aliases +encodings.ascii +encodings.base64_codec +encodings.big5 +encodings.big5hkscs +encodings.bz2_codec +encodings.charmap +encodings.cp037 +encodings.cp1006 +encodings.cp1026 +encodings.cp1125 +encodings.cp1140 +encodings.cp1250 +encodings.cp1251 +encodings.cp1252 +encodings.cp1253 +encodings.cp1254 +encodings.cp1255 +encodings.cp1256 +encodings.cp1257 +encodings.cp1258 +encodings.cp273 +encodings.cp424 +encodings.cp437 +encodings.cp500 +encodings.cp720 +encodings.cp737 +encodings.cp775 +encodings.cp850 +encodings.cp852 +encodings.cp855 +encodings.cp856 +encodings.cp857 +encodings.cp858 +encodings.cp860 +encodings.cp861 +encodings.cp862 +encodings.cp863 +encodings.cp864 +encodings.cp865 +encodings.cp866 +encodings.cp869 +encodings.cp874 +encodings.cp875 +encodings.cp932 +encodings.cp949 +encodings.cp950 +encodings.euc_jis_2004 +encodings.euc_jisx0213 +encodings.euc_jp +encodings.euc_kr +encodings.gb18030 +encodings.gb2312 +encodings.gbk +encodings.hex_codec +encodings.hp_roman8 +encodings.hz +encodings.idna +encodings.iso2022_jp +encodings.iso2022_jp_1 +encodings.iso2022_jp_2 +encodings.iso2022_jp_2004 +encodings.iso2022_jp_3 +encodings.iso2022_jp_ext +encodings.iso2022_kr +encodings.iso8859_1 +encodings.iso8859_10 +encodings.iso8859_11 +encodings.iso8859_13 +encodings.iso8859_14 +encodings.iso8859_15 +encodings.iso8859_16 +encodings.iso8859_2 +encodings.iso8859_3 +encodings.iso8859_4 +encodings.iso8859_5 +encodings.iso8859_6 +encodings.iso8859_7 +encodings.iso8859_8 +encodings.iso8859_9 +encodings.johab +encodings.koi8_r +encodings.koi8_t +encodings.koi8_u +encodings.kz1048 +encodings.latin_1 +encodings.mac_arabic +encodings.mac_croatian +encodings.mac_cyrillic +encodings.mac_farsi +encodings.mac_greek +encodings.mac_iceland +encodings.mac_latin2 +encodings.mac_roman +encodings.mac_romanian +encodings.mac_turkish +encodings.mbcs +encodings.oem +encodings.palmos +encodings.ptcp154 +encodings.punycode +encodings.quopri_codec +encodings.raw_unicode_escape +encodings.rot_13 +encodings.shift_jis +encodings.shift_jis_2004 +encodings.shift_jisx0213 +encodings.tis_620 +encodings.undefined +encodings.unicode_escape +encodings.utf_16 +encodings.utf_16_be +encodings.utf_16_le +encodings.utf_32 +encodings.utf_32_be +encodings.utf_32_le +encodings.utf_7 +encodings.utf_8 +encodings.utf_8_sig +encodings.uu_codec +encodings.zlib_codec +ensurepip +ensurepip.__main__ +ensurepip._uninstall +enum +errno +faulthandler +fcntl +filecmp +fileinput +fnmatch +fractions +ftplib +functools +gc +genericpath +getopt +getpass +gettext +glob +graphlib +grp +gzip +hashlib +heapq +hmac +html +html.entities +html.parser +http +http.client +http.cookiejar +http.cookies +http.server +idlelib +idlelib.__main__ +idlelib.autocomplete +idlelib.autocomplete_w +idlelib.autoexpand +idlelib.browser +idlelib.calltip +idlelib.calltip_w +idlelib.codecontext +idlelib.colorizer +idlelib.config +idlelib.config_key +idlelib.configdialog +idlelib.debugger +idlelib.debugger_r +idlelib.debugobj +idlelib.debugobj_r +idlelib.delegator +idlelib.dynoption +idlelib.editor +idlelib.filelist +idlelib.format +idlelib.grep +idlelib.help +idlelib.help_about +idlelib.history +idlelib.hyperparser +idlelib.idle +idlelib.idle_test +idlelib.idle_test.htest +idlelib.idle_test.mock_idle +idlelib.idle_test.mock_tk +idlelib.idle_test.template +idlelib.idle_test.test_autocomplete +idlelib.idle_test.test_autocomplete_w +idlelib.idle_test.test_autoexpand +idlelib.idle_test.test_browser +idlelib.idle_test.test_calltip +idlelib.idle_test.test_calltip_w +idlelib.idle_test.test_codecontext +idlelib.idle_test.test_colorizer +idlelib.idle_test.test_config +idlelib.idle_test.test_config_key +idlelib.idle_test.test_configdialog +idlelib.idle_test.test_debugger +idlelib.idle_test.test_debugger_r +idlelib.idle_test.test_debugobj +idlelib.idle_test.test_debugobj_r +idlelib.idle_test.test_delegator +idlelib.idle_test.test_editmenu +idlelib.idle_test.test_editor +idlelib.idle_test.test_filelist +idlelib.idle_test.test_format +idlelib.idle_test.test_grep +idlelib.idle_test.test_help +idlelib.idle_test.test_help_about +idlelib.idle_test.test_history +idlelib.idle_test.test_hyperparser +idlelib.idle_test.test_iomenu +idlelib.idle_test.test_macosx +idlelib.idle_test.test_mainmenu +idlelib.idle_test.test_multicall +idlelib.idle_test.test_outwin +idlelib.idle_test.test_parenmatch +idlelib.idle_test.test_pathbrowser +idlelib.idle_test.test_percolator +idlelib.idle_test.test_pyparse +idlelib.idle_test.test_pyshell +idlelib.idle_test.test_query +idlelib.idle_test.test_redirector +idlelib.idle_test.test_replace +idlelib.idle_test.test_rpc +idlelib.idle_test.test_run +idlelib.idle_test.test_runscript +idlelib.idle_test.test_scrolledlist +idlelib.idle_test.test_search +idlelib.idle_test.test_searchbase +idlelib.idle_test.test_searchengine +idlelib.idle_test.test_sidebar +idlelib.idle_test.test_squeezer +idlelib.idle_test.test_stackviewer +idlelib.idle_test.test_statusbar +idlelib.idle_test.test_text +idlelib.idle_test.test_textview +idlelib.idle_test.test_tooltip +idlelib.idle_test.test_tree +idlelib.idle_test.test_undo +idlelib.idle_test.test_util +idlelib.idle_test.test_warning +idlelib.idle_test.test_window +idlelib.idle_test.test_zoomheight +idlelib.idle_test.test_zzdummy +idlelib.idle_test.tkinter_testing_utils +idlelib.iomenu +idlelib.macosx +idlelib.mainmenu +idlelib.multicall +idlelib.outwin +idlelib.parenmatch +idlelib.pathbrowser +idlelib.percolator +idlelib.pyparse +idlelib.pyshell +idlelib.query +idlelib.redirector +idlelib.replace +idlelib.rpc +idlelib.run +idlelib.runscript +idlelib.scrolledlist +idlelib.search +idlelib.searchbase +idlelib.searchengine +idlelib.sidebar +idlelib.squeezer +idlelib.stackviewer +idlelib.statusbar +idlelib.textview +idlelib.tooltip +idlelib.tree +idlelib.undo +idlelib.util +idlelib.window +idlelib.zoomheight +idlelib.zzdummy +imaplib +importlib +importlib._abc +importlib._bootstrap +importlib._bootstrap_external +importlib.abc +importlib.machinery +importlib.metadata +importlib.metadata._adapters +importlib.metadata._collections +importlib.metadata._functools +importlib.metadata._itertools +importlib.metadata._meta +importlib.metadata._text +importlib.metadata.diagnose +importlib.readers +importlib.resources +importlib.resources._adapters +importlib.resources._common +importlib.resources._functional +importlib.resources._itertools +importlib.resources.abc +importlib.resources.readers +importlib.resources.simple +importlib.simple +importlib.util +inspect +io +ipaddress +itertools +json +json.__main__ +json.decoder +json.encoder +json.scanner +json.tool +keyword +linecache +locale +logging +logging.config +logging.handlers +lzma +mailbox +marshal +math +mimetypes +mmap +modulefinder +msvcrt +multiprocessing +multiprocessing.connection +multiprocessing.context +multiprocessing.dummy +multiprocessing.dummy.connection +multiprocessing.forkserver +multiprocessing.heap +multiprocessing.managers +multiprocessing.pool +multiprocessing.popen_fork +multiprocessing.popen_forkserver +multiprocessing.popen_spawn_posix +multiprocessing.popen_spawn_win32 +multiprocessing.process +multiprocessing.queues +multiprocessing.reduction +multiprocessing.resource_sharer +multiprocessing.resource_tracker +multiprocessing.shared_memory +multiprocessing.sharedctypes +multiprocessing.spawn +multiprocessing.synchronize +multiprocessing.util +netrc +nt +ntpath +nturl2path +numbers +opcode +operator +optparse +os +os.path +pathlib +pathlib._os +pathlib.types +pdb +pickle +pickletools +pkgutil +platform +plistlib +poplib +posix +posixpath +pprint +profile +pstats +pty +pwd +py_compile +pyclbr +pydoc +pydoc_data +pydoc_data.topics +pyexpat +pyexpat.errors +pyexpat.model +queue +quopri +random +re +re._casefix +re._compiler +re._constants +re._parser +readline +reprlib +resource +rlcompleter +runpy +sched +secrets +select +selectors +shelve +shlex +shutil +signal +site +smtplib +socket +socketserver +sqlite3 +sqlite3.__main__ +sqlite3.dbapi2 +sqlite3.dump +sre_compile +sre_constants +sre_parse +ssl +stat +statistics +string +string.templatelib +stringprep +struct +subprocess +symtable +sys +sysconfig +sysconfig.__main__ +syslog +tabnanny +tarfile +tempfile +termios +textwrap +this +threading +time +timeit +tkinter +tkinter.__main__ +tkinter.colorchooser +tkinter.commondialog +tkinter.constants +tkinter.dialog +tkinter.dnd +tkinter.filedialog +tkinter.font +tkinter.messagebox +tkinter.scrolledtext +tkinter.simpledialog +tkinter.ttk +token +tokenize +tomllib +tomllib._parser +tomllib._re +tomllib._types +trace +traceback +tracemalloc +tty +turtle +turtledemo +turtledemo.__main__ +turtledemo.bytedesign +turtledemo.chaos +turtledemo.clock +turtledemo.colormixer +turtledemo.forest +turtledemo.fractalcurves +turtledemo.lindenmayer +turtledemo.minimal_hanoi +turtledemo.nim +turtledemo.paint +turtledemo.peace +turtledemo.penrose +turtledemo.planet_and_moon +turtledemo.rosette +turtledemo.round_dance +turtledemo.sorting_animate +turtledemo.tree +turtledemo.two_canvases +turtledemo.yinyang +types +typing +unicodedata +unittest +unittest.__main__ +unittest._log +unittest.async_case +unittest.case +unittest.loader +unittest.main +unittest.mock +unittest.result +unittest.runner +unittest.signals +unittest.suite +unittest.util +urllib +urllib.error +urllib.parse +urllib.request +urllib.response +urllib.robotparser +uuid +venv +venv.__main__ +warnings +wave +weakref +webbrowser +winreg +winsound +wsgiref +wsgiref.handlers +wsgiref.headers +wsgiref.simple_server +wsgiref.types +wsgiref.util +wsgiref.validate +xml +xml.dom +xml.dom.NodeFilter +xml.dom.domreg +xml.dom.expatbuilder +xml.dom.minicompat +xml.dom.minidom +xml.dom.pulldom +xml.dom.xmlbuilder +xml.etree +xml.etree.ElementInclude +xml.etree.ElementPath +xml.etree.ElementTree +xml.etree.cElementTree +xml.parsers +xml.parsers.expat +xml.sax +xml.sax._exceptions +xml.sax.expatreader +xml.sax.handler +xml.sax.saxutils +xml.sax.xmlreader +xmlrpc +xmlrpc.client +xmlrpc.server +zipapp +zipfile +zipfile.__main__ +zipfile._path +zipfile._path.glob +zipimport +zlib +zoneinfo +zoneinfo._common +zoneinfo._tzpath +zoneinfo._zoneinfo
diff --git a/news/3821.fixed.md b/news/3821.fixed.md new file mode 100644 index 0000000..a880d13 --- /dev/null +++ b/news/3821.fixed.md
@@ -0,0 +1,5 @@ +(gazelle) Check in a default standard library module list (`stdlib_list/default.txt`) +alongside the Bazel-selected list (`stdlib_list/selected.txt`) so +`github.com/bazel-contrib/rules_python/gazelle/python` can be built and tested +directly with standard Go tooling (`go build`, `go test`, `go list`) outside +Bazel ([#3821](https://github.com/bazel-contrib/rules_python/issues/3821)).