From 5594d3a5da15d04a3788030bd01279c13bb019dc Mon Sep 17 00:00:00 2001 From: Alex Hornby Date: Mon, 6 Jan 2025 11:16:30 -0800 Subject: [PATCH] skip unnecessary github actions steps Summary: X-link: https://github.com/facebookincubator/zstrong/pull/1100 Update generated github actions to only run the fetch and and build steps when there are sources expected for a manifest For local github actions testing using `act` this speeds up the test runs, and in real github CI it makes it clearer which steps are actually doing something on the given runner (we don't know exactly what it has installed beforehand) Also set the windows git config the same as on internal CI Reviewed By: bigfootjon Differential Revision: D67839708 fbshipit-source-id: 0a60c6fc89e8c6abb2464f879459aa23d5aec969 --- .github/workflows/edencommon_linux.yml | 77 +++++++++++++++- build/fbcode_builder/getdeps.py | 111 ++++++++++++++++-------- build/fbcode_builder/getdeps/fetcher.py | 4 +- 3 files changed, 154 insertions(+), 38 deletions(-) diff --git a/.github/workflows/edencommon_linux.yml b/.github/workflows/edencommon_linux.yml index 3a9774c..aa0f809 100644 --- a/.github/workflows/edencommon_linux.yml +++ b/.github/workflows/edencommon_linux.yml @@ -27,148 +27,219 @@ jobs: - name: Update system package info run: sudo apt-get update - name: Install system deps - run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive edencommon - - name: Install packaging system deps - run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive patchelf + run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive edencommon && sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive patchelf + - id: paths + name: Query paths + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages query-paths --recursive --src-dir=. edencommon >> "$GITHUB_OUTPUT" - name: Fetch ninja + if: ${{ steps.paths.outputs.ninja_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests ninja - name: Fetch cmake + if: ${{ steps.paths.outputs.cmake_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests cmake - name: Fetch gflags + if: ${{ steps.paths.outputs.gflags_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests gflags - name: Fetch glog + if: ${{ steps.paths.outputs.glog_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests glog - name: Fetch fmt + if: ${{ steps.paths.outputs.fmt_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fmt - name: Fetch googletest + if: ${{ steps.paths.outputs.googletest_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests googletest - name: Fetch xxhash + if: ${{ steps.paths.outputs.xxhash_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests xxhash - name: Fetch zstd + if: ${{ steps.paths.outputs.zstd_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests zstd - name: Fetch boost + if: ${{ steps.paths.outputs.boost_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests boost - name: Fetch double-conversion + if: ${{ steps.paths.outputs.double-conversion_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests double-conversion - name: Fetch fast_float + if: ${{ steps.paths.outputs.fast_float_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fast_float - name: Fetch libdwarf + if: ${{ steps.paths.outputs.libdwarf_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libdwarf - name: Fetch libevent + if: ${{ steps.paths.outputs.libevent_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libevent - name: Fetch lz4 + if: ${{ steps.paths.outputs.lz4_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests lz4 - name: Fetch snappy + if: ${{ steps.paths.outputs.snappy_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests snappy - name: Fetch zlib + if: ${{ steps.paths.outputs.zlib_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests zlib - name: Fetch bz2 + if: ${{ steps.paths.outputs.bz2_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests bz2 - name: Fetch openssl + if: ${{ steps.paths.outputs.openssl_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests openssl - name: Fetch liboqs + if: ${{ steps.paths.outputs.liboqs_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests liboqs - name: Fetch autoconf + if: ${{ steps.paths.outputs.autoconf_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests autoconf - name: Fetch automake + if: ${{ steps.paths.outputs.automake_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests automake - name: Fetch libtool + if: ${{ steps.paths.outputs.libtool_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libtool - name: Fetch libsodium + if: ${{ steps.paths.outputs.libsodium_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libsodium - name: Fetch libiberty + if: ${{ steps.paths.outputs.libiberty_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libiberty - name: Fetch libunwind + if: ${{ steps.paths.outputs.libunwind_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libunwind - name: Fetch xz + if: ${{ steps.paths.outputs.xz_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests xz - name: Fetch folly + if: ${{ steps.paths.outputs.folly_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests folly - name: Fetch fizz + if: ${{ steps.paths.outputs.fizz_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fizz - name: Fetch mvfst + if: ${{ steps.paths.outputs.mvfst_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests mvfst - name: Fetch libffi + if: ${{ steps.paths.outputs.libffi_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libffi - name: Fetch ncurses + if: ${{ steps.paths.outputs.ncurses_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests ncurses - name: Fetch python + if: ${{ steps.paths.outputs.python_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests python - name: Fetch wangle + if: ${{ steps.paths.outputs.wangle_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests wangle - name: Fetch fbthrift + if: ${{ steps.paths.outputs.fbthrift_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fbthrift - name: Fetch fb303 + if: ${{ steps.paths.outputs.fb303_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fb303 - name: Build ninja + if: ${{ steps.paths.outputs.ninja_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests ninja - name: Build cmake + if: ${{ steps.paths.outputs.cmake_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests cmake - name: Build gflags + if: ${{ steps.paths.outputs.gflags_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests gflags - name: Build glog + if: ${{ steps.paths.outputs.glog_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests glog - name: Build fmt + if: ${{ steps.paths.outputs.fmt_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fmt - name: Build googletest + if: ${{ steps.paths.outputs.googletest_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests googletest - name: Build xxhash + if: ${{ steps.paths.outputs.xxhash_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests xxhash - name: Build zstd + if: ${{ steps.paths.outputs.zstd_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests zstd - name: Build boost + if: ${{ steps.paths.outputs.boost_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests boost - name: Build double-conversion + if: ${{ steps.paths.outputs.double-conversion_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests double-conversion - name: Build fast_float + if: ${{ steps.paths.outputs.fast_float_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fast_float - name: Build libdwarf + if: ${{ steps.paths.outputs.libdwarf_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libdwarf - name: Build libevent + if: ${{ steps.paths.outputs.libevent_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libevent - name: Build lz4 + if: ${{ steps.paths.outputs.lz4_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests lz4 - name: Build snappy + if: ${{ steps.paths.outputs.snappy_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests snappy - name: Build zlib + if: ${{ steps.paths.outputs.zlib_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests zlib - name: Build bz2 + if: ${{ steps.paths.outputs.bz2_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests bz2 - name: Build openssl + if: ${{ steps.paths.outputs.openssl_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests openssl - name: Build liboqs + if: ${{ steps.paths.outputs.liboqs_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests liboqs - name: Build autoconf + if: ${{ steps.paths.outputs.autoconf_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests autoconf - name: Build automake + if: ${{ steps.paths.outputs.automake_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests automake - name: Build libtool + if: ${{ steps.paths.outputs.libtool_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libtool - name: Build libsodium + if: ${{ steps.paths.outputs.libsodium_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libsodium - name: Build libiberty + if: ${{ steps.paths.outputs.libiberty_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libiberty - name: Build libunwind + if: ${{ steps.paths.outputs.libunwind_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libunwind - name: Build xz + if: ${{ steps.paths.outputs.xz_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests xz - name: Build folly + if: ${{ steps.paths.outputs.folly_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests folly - name: Build fizz + if: ${{ steps.paths.outputs.fizz_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fizz - name: Build mvfst + if: ${{ steps.paths.outputs.mvfst_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests mvfst - name: Build libffi + if: ${{ steps.paths.outputs.libffi_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libffi - name: Build ncurses + if: ${{ steps.paths.outputs.ncurses_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests ncurses - name: Build python + if: ${{ steps.paths.outputs.python_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests python - name: Build wangle + if: ${{ steps.paths.outputs.wangle_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests wangle - name: Build fbthrift + if: ${{ steps.paths.outputs.fbthrift_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fbthrift - name: Build fb303 + if: ${{ steps.paths.outputs.fb303_SOURCE }} run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fb303 - name: Build edencommon run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. edencommon --project-install-prefix edencommon:/usr/local diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index 03a5f3c..138e729 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -552,6 +552,33 @@ def setup_project_cmd_parser(self, parser): ) +@cmd("query-paths", "print the paths for tooling to use") +class QueryPathsCmd(ProjectCmdBase): + def run_project_cmd(self, args, loader, manifest): + if args.recursive: + manifests = loader.manifests_in_dependency_order() + else: + manifests = [manifest] + + for m in manifests: + fetcher = loader.create_fetcher(m) + if isinstance(fetcher, SystemPackageFetcher): + # We are guaranteed that if the fetcher is set to + # SystemPackageFetcher then this item is completely + # satisfied by the appropriate system packages + continue + src_dir = fetcher.get_src_dir() + print(f"{m.name}_SOURCE={src_dir}") + + def setup_project_cmd_parser(self, parser): + parser.add_argument( + "--recursive", + help="print the transitive deps also", + action="store_true", + default=False, + ) + + @cmd("show-source-dir", "print the source dir for a given project") class ShowSourceDirCmd(ProjectCmdBase): def run_project_cmd(self, args, loader, manifest): @@ -1001,6 +1028,10 @@ def write_job_for_platform(self, platform, args): # noqa: C901 manifest_ctx.set("test", "on") run_on = self.get_run_on(args) + tests_arg = "--no-tests " + if run_tests: + tests_arg = "" + # Some projects don't do anything "useful" as a leaf project, only # as a dep for a leaf project. Check for those here; we don't want # to waste the effort scheduling them on CI. @@ -1086,12 +1117,14 @@ def write_job_for_platform(self, platform, args): # noqa: C901 ) out.write(" shell: cmd\n") - # The git installation may not like long filenames, so tell it - # that we want it to use them! out.write(" - name: Fix Git config\n") - out.write(" run: git config --system core.longpaths true\n") - out.write(" - name: Disable autocrlf\n") - out.write(" run: git config --system core.autocrlf false\n") + out.write(" run: >\n") + out.write(" git config --system core.longpaths true &&\n") + out.write(" git config --system core.autocrlf false &&\n") + # cxx crate needs symlinks enabled + out.write(" git config --system core.symlinks true\n") + # && is not supported on default windows powershell, so use cmd + out.write(" shell: cmd\n") out.write(" - uses: actions/checkout@v4\n") @@ -1127,17 +1160,12 @@ def write_job_for_platform(self, platform, args): # noqa: C901 if build_opts.is_darwin(): # brew is installed as regular user sudo_arg = "" - tests_arg = "--no-tests " - if run_tests: - tests_arg = "" - out.write( - f" run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps {tests_arg}--recursive {manifest.name}\n" - ) + + system_deps_cmd = f"{sudo_arg}{getdepscmd}{allow_sys_arg} install-system-deps {tests_arg}--recursive {manifest.name}" if build_opts.is_linux() or build_opts.is_freebsd(): - out.write(" - name: Install packaging system deps\n") - out.write( - f" run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps {tests_arg}--recursive patchelf\n" - ) + system_deps_cmd += f" && {sudo_arg}{getdepscmd}{allow_sys_arg} install-system-deps {tests_arg}--recursive patchelf" + out.write(f" run: {system_deps_cmd}\n") + required_locales = manifest.get( "github.actions", "required_locales", ctx=manifest_ctx ) @@ -1152,6 +1180,18 @@ def write_job_for_platform(self, platform, args): # noqa: C901 out.write(f" - name: Ensure {loc} locale present\n") out.write(f" run: {sudo_arg}locale-gen {loc}\n") + out.write(" - id: paths\n") + out.write(" name: Query paths\n") + if build_opts.is_windows(): + out.write( + f" run: {getdepscmd}{allow_sys_arg} query-paths {tests_arg}--recursive --src-dir=. {manifest.name} >> $env:GITHUB_OUTPUT\n" + ) + out.write(" shell: pwsh\n") + else: + out.write( + f' run: {getdepscmd}{allow_sys_arg} query-paths {tests_arg}--recursive --src-dir=. {manifest.name} >> "$GITHUB_OUTPUT"\n' + ) + projects = loader.manifests_in_dependency_order() main_repo_url = manifest.get_repo_url(manifest_ctx) @@ -1178,25 +1218,32 @@ def write_job_for_platform(self, platform, args): # noqa: C901 ctx = loader.ctx_gen.get_context(m.name) if m.get_repo_url(ctx) != main_repo_url: out.write(" - name: Fetch %s\n" % m.name) + out.write( + f" if: ${{{{ steps.paths.outputs.{m.name}_SOURCE }}}}\n" + ) out.write( f" run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n" ) for m in projects: - if m != manifest: - if m.name == "rust": - continue - else: - src_dir_arg = "" - ctx = loader.ctx_gen.get_context(m.name) - if main_repo_url and m.get_repo_url(ctx) == main_repo_url: - # Its in the same repo, so src-dir is also . - src_dir_arg = "--src-dir=. " - has_same_repo_dep = True - out.write(" - name: Build %s\n" % m.name) - out.write( - f" run: {getdepscmd}{allow_sys_arg} build {build_type_arg}{src_dir_arg}{free_up_disk}--no-tests {m.name}\n" - ) + if m == manifest or m.name == "rust": + continue + src_dir_arg = "" + ctx = loader.ctx_gen.get_context(m.name) + if main_repo_url and m.get_repo_url(ctx) == main_repo_url: + # Its in the same repo, so src-dir is also . + src_dir_arg = "--src-dir=. " + has_same_repo_dep = True + + out.write(" - name: Build %s\n" % m.name) + if not src_dir_arg: + # only run the step if needed + out.write( + f" if: ${{{{ steps.paths.outputs.{m.name}_SOURCE }}}}\n" + ) + out.write( + f" run: {getdepscmd}{allow_sys_arg} build {build_type_arg}{src_dir_arg}{free_up_disk}--no-tests {m.name}\n" + ) out.write(" - name: Build %s\n" % manifest.name) @@ -1213,12 +1260,8 @@ def write_job_for_platform(self, platform, args): # noqa: C901 if has_same_repo_dep: no_deps_arg = "--no-deps " - no_tests_arg = "" - if not run_tests: - no_tests_arg = "--no-tests " - out.write( - f" run: {getdepscmd}{allow_sys_arg} build {build_type_arg}{no_tests_arg}{no_deps_arg}--src-dir=. {manifest.name} {project_prefix}\n" + f" run: {getdepscmd}{allow_sys_arg} build {build_type_arg}{tests_arg}{no_deps_arg}--src-dir=. {manifest.name} {project_prefix}\n" ) out.write(" - name: Copy artifacts\n") diff --git a/build/fbcode_builder/getdeps/fetcher.py b/build/fbcode_builder/getdeps/fetcher.py index c769c11..a173708 100644 --- a/build/fbcode_builder/getdeps/fetcher.py +++ b/build/fbcode_builder/getdeps/fetcher.py @@ -244,7 +244,9 @@ def __init__(self, build_options, manifest, repo_url, rev, depth) -> None: if not m: raise Exception("Failed to parse rev from %s" % hash_file) rev = m.group(1) - print("Using pinned rev %s for %s" % (rev, repo_url)) + print( + "Using pinned rev %s for %s" % (rev, repo_url), file=sys.stderr + ) self.rev = rev or "main" self.origin_repo = repo_url