You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
601 lines
18 KiB
601 lines
18 KiB
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
# Copyright 2020 The ChromiumOS Authors
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
"""Get an upstream patch to LLVM's PATCHES.json."""
|
|
|
|
import argparse
|
|
import dataclasses
|
|
from datetime import datetime
|
|
import json
|
|
import logging
|
|
import os
|
|
from pathlib import Path
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
import typing as t
|
|
|
|
import chroot
|
|
import get_llvm_hash
|
|
import git
|
|
import git_llvm_rev
|
|
import patch_utils
|
|
import update_chromeos_llvm_hash
|
|
|
|
|
|
__DOC_EPILOGUE = """
|
|
Example Usage:
|
|
get_upstream_patch --chroot_path ~/chromiumos --platform chromiumos \
|
|
--sha 1234567 --sha 890abdc
|
|
"""
|
|
|
|
|
|
class CherrypickError(ValueError):
|
|
"""A ValueError that highlights the cherry-pick has been seen before"""
|
|
|
|
|
|
class CherrypickVersionError(ValueError):
|
|
"""A ValueError that highlights the cherry-pick is before the start_sha"""
|
|
|
|
|
|
class PatchApplicationError(ValueError):
|
|
"""A ValueError indicating that a test patch application was unsuccessful"""
|
|
|
|
|
|
def validate_patch_application(
|
|
llvm_dir: Path, svn_version: int, patches_json_fp: Path, patch_props
|
|
):
|
|
|
|
start_sha = get_llvm_hash.GetGitHashFrom(llvm_dir, svn_version)
|
|
subprocess.run(["git", "-C", llvm_dir, "checkout", start_sha], check=True)
|
|
|
|
predecessor_apply_results = patch_utils.apply_all_from_json(
|
|
svn_version, llvm_dir, patches_json_fp, continue_on_failure=True
|
|
)
|
|
|
|
if predecessor_apply_results.failed_patches:
|
|
logging.error("Failed to apply patches from PATCHES.json:")
|
|
for p in predecessor_apply_results.failed_patches:
|
|
logging.error(f"Patch title: {p.title()}")
|
|
raise PatchApplicationError("Failed to apply patch from PATCHES.json")
|
|
|
|
patch_entry = patch_utils.PatchEntry.from_dict(
|
|
patches_json_fp.parent, patch_props
|
|
)
|
|
test_apply_result = patch_entry.test_apply(Path(llvm_dir))
|
|
|
|
if not test_apply_result:
|
|
logging.error("Could not apply requested patch")
|
|
logging.error(test_apply_result.failure_info())
|
|
raise PatchApplicationError(
|
|
f'Failed to apply patch: {patch_props["metadata"]["title"]}'
|
|
)
|
|
|
|
|
|
def add_patch(
|
|
patches_json_path: str,
|
|
patches_dir: str,
|
|
relative_patches_dir: str,
|
|
start_version: git_llvm_rev.Rev,
|
|
llvm_dir: str,
|
|
rev: t.Union[git_llvm_rev.Rev, str],
|
|
sha: str,
|
|
package: str,
|
|
platforms: t.List[str],
|
|
):
|
|
"""Gets the start and end intervals in 'json_file'.
|
|
|
|
Args:
|
|
patches_json_path: The absolute path to PATCHES.json.
|
|
patches_dir: The aboslute path to the directory patches are in.
|
|
relative_patches_dir: The relative path to PATCHES.json.
|
|
start_version: The base LLVM revision this patch applies to.
|
|
llvm_dir: The path to LLVM checkout.
|
|
rev: An LLVM revision (git_llvm_rev.Rev) for a cherrypicking, or a
|
|
differential revision (str) otherwise.
|
|
sha: The LLVM git sha that corresponds to the patch. For differential
|
|
revisions, the git sha from the local commit created by 'arc patch'
|
|
is used.
|
|
package: The LLVM project name this patch applies to.
|
|
platforms: List of platforms this patch applies to.
|
|
|
|
Raises:
|
|
CherrypickError: A ValueError that highlights the cherry-pick has been
|
|
seen before.
|
|
CherrypickRangeError: A ValueError that's raised when the given patch
|
|
is from before the start_sha.
|
|
"""
|
|
|
|
is_cherrypick = isinstance(rev, git_llvm_rev.Rev)
|
|
if is_cherrypick:
|
|
file_name = f"{sha}.patch"
|
|
else:
|
|
file_name = f"{rev}.patch"
|
|
rel_patch_path = os.path.join(relative_patches_dir, file_name)
|
|
|
|
# Check that we haven't grabbed a patch range that's nonsensical.
|
|
end_vers = rev.number if isinstance(rev, git_llvm_rev.Rev) else None
|
|
if end_vers is not None and end_vers <= start_version.number:
|
|
raise CherrypickVersionError(
|
|
f"`until` version {end_vers} is earlier or equal to"
|
|
f" `from` version {start_version.number} for patch"
|
|
f" {rel_patch_path}"
|
|
)
|
|
|
|
with open(patches_json_path, encoding="utf-8") as f:
|
|
patches_json = json.load(f)
|
|
|
|
for p in patches_json:
|
|
rel_path = p["rel_patch_path"]
|
|
if rel_path == rel_patch_path:
|
|
raise CherrypickError(
|
|
f"Patch at {rel_path} already exists in PATCHES.json"
|
|
)
|
|
if is_cherrypick:
|
|
if sha in rel_path:
|
|
logging.warning(
|
|
"Similarly-named patch already exists in PATCHES.json: %r",
|
|
rel_path,
|
|
)
|
|
|
|
with open(os.path.join(patches_dir, file_name), "wb") as f:
|
|
cmd = ["git", "show", sha]
|
|
# Only apply the part of the patch that belongs to this package, expect
|
|
# LLVM. This is because some packages are built with LLVM ebuild on X86 but
|
|
# not on the other architectures. e.g. compiler-rt. Therefore always apply
|
|
# the entire patch to LLVM ebuild as a workaround.
|
|
if package != "llvm":
|
|
cmd.append(package_to_project(package))
|
|
subprocess.check_call(cmd, stdout=f, cwd=llvm_dir)
|
|
|
|
commit_subject = subprocess.check_output(
|
|
["git", "log", "-n1", "--format=%s", sha],
|
|
cwd=llvm_dir,
|
|
encoding="utf-8",
|
|
)
|
|
patch_props = {
|
|
"rel_patch_path": rel_patch_path,
|
|
"metadata": {
|
|
"title": commit_subject.strip(),
|
|
"info": [],
|
|
},
|
|
"platforms": sorted(platforms),
|
|
"version_range": {
|
|
"from": start_version.number,
|
|
"until": end_vers,
|
|
},
|
|
}
|
|
|
|
with patch_utils.git_clean_context(Path(llvm_dir)):
|
|
validate_patch_application(
|
|
Path(llvm_dir),
|
|
start_version.number,
|
|
Path(patches_json_path),
|
|
patch_props,
|
|
)
|
|
|
|
patches_json.append(patch_props)
|
|
|
|
temp_file = patches_json_path + ".tmp"
|
|
with open(temp_file, "w", encoding="utf-8") as f:
|
|
json.dump(
|
|
patches_json, f, indent=4, separators=(",", ": "), sort_keys=True
|
|
)
|
|
f.write("\n")
|
|
os.rename(temp_file, patches_json_path)
|
|
|
|
|
|
def parse_ebuild_for_assignment(ebuild_path: str, var_name: str) -> str:
|
|
# '_pre' filters the LLVM 9.0 ebuild, which we never want to target, from
|
|
# this list.
|
|
candidates = [
|
|
x
|
|
for x in os.listdir(ebuild_path)
|
|
if x.endswith(".ebuild") and "_pre" in x
|
|
]
|
|
|
|
if not candidates:
|
|
raise ValueError("No ebuilds found under %r" % ebuild_path)
|
|
|
|
ebuild = os.path.join(ebuild_path, max(candidates))
|
|
with open(ebuild, encoding="utf-8") as f:
|
|
var_name_eq = var_name + "="
|
|
for orig_line in f:
|
|
if not orig_line.startswith(var_name_eq):
|
|
continue
|
|
|
|
# We shouldn't see much variety here, so do the simplest thing possible.
|
|
line = orig_line[len(var_name_eq) :]
|
|
# Remove comments
|
|
line = line.split("#")[0]
|
|
# Remove quotes
|
|
line = shlex.split(line)
|
|
if len(line) != 1:
|
|
raise ValueError(
|
|
"Expected exactly one quoted value in %r" % orig_line
|
|
)
|
|
return line[0].strip()
|
|
|
|
raise ValueError("No %s= line found in %r" % (var_name, ebuild))
|
|
|
|
|
|
# Resolves a git ref (or similar) to a LLVM SHA.
|
|
def resolve_llvm_ref(llvm_dir: str, sha: str) -> str:
|
|
return subprocess.check_output(
|
|
["git", "rev-parse", sha],
|
|
encoding="utf-8",
|
|
cwd=llvm_dir,
|
|
).strip()
|
|
|
|
|
|
# Get the package name of an LLVM project
|
|
def project_to_package(project: str) -> str:
|
|
if project == "libunwind":
|
|
return "llvm-libunwind"
|
|
return project
|
|
|
|
|
|
# Get the LLVM project name of a package
|
|
def package_to_project(package: str) -> str:
|
|
if package == "llvm-libunwind":
|
|
return "libunwind"
|
|
return package
|
|
|
|
|
|
# Get the LLVM projects change in the specifed sha
|
|
def get_package_names(sha: str, llvm_dir: str) -> list:
|
|
paths = subprocess.check_output(
|
|
["git", "show", "--name-only", "--format=", sha],
|
|
cwd=llvm_dir,
|
|
encoding="utf-8",
|
|
).splitlines()
|
|
# Some LLVM projects are built by LLVM ebuild on X86, so always apply the
|
|
# patch to LLVM ebuild
|
|
packages = {"llvm"}
|
|
# Detect if there are more packages to apply the patch to
|
|
for path in paths:
|
|
package = project_to_package(path.split("/")[0])
|
|
if package in ("compiler-rt", "libcxx", "libcxxabi", "llvm-libunwind"):
|
|
packages.add(package)
|
|
packages = list(sorted(packages))
|
|
return packages
|
|
|
|
|
|
def create_patch_for_packages(
|
|
packages: t.List[str],
|
|
symlinks: t.List[str],
|
|
start_rev: git_llvm_rev.Rev,
|
|
rev: t.Union[git_llvm_rev.Rev, str],
|
|
sha: str,
|
|
llvm_dir: str,
|
|
platforms: t.List[str],
|
|
):
|
|
"""Create a patch and add its metadata for each package"""
|
|
for package, symlink in zip(packages, symlinks):
|
|
symlink_dir = os.path.dirname(symlink)
|
|
patches_json_path = os.path.join(symlink_dir, "files/PATCHES.json")
|
|
relative_patches_dir = "cherry" if package == "llvm" else ""
|
|
patches_dir = os.path.join(symlink_dir, "files", relative_patches_dir)
|
|
logging.info("Getting %s (%s) into %s", rev, sha, package)
|
|
add_patch(
|
|
patches_json_path,
|
|
patches_dir,
|
|
relative_patches_dir,
|
|
start_rev,
|
|
llvm_dir,
|
|
rev,
|
|
sha,
|
|
package,
|
|
platforms=platforms,
|
|
)
|
|
|
|
|
|
def make_cl(
|
|
symlinks_to_uprev: t.List[str],
|
|
llvm_symlink_dir: str,
|
|
branch: str,
|
|
commit_messages: t.List[str],
|
|
reviewers: t.Optional[t.List[str]],
|
|
cc: t.Optional[t.List[str]],
|
|
):
|
|
symlinks_to_uprev = sorted(set(symlinks_to_uprev))
|
|
for symlink in symlinks_to_uprev:
|
|
update_chromeos_llvm_hash.UprevEbuildSymlink(symlink)
|
|
subprocess.check_output(
|
|
["git", "add", "--all"], cwd=os.path.dirname(symlink)
|
|
)
|
|
git.UploadChanges(llvm_symlink_dir, branch, commit_messages, reviewers, cc)
|
|
git.DeleteBranch(llvm_symlink_dir, branch)
|
|
|
|
|
|
def resolve_symbolic_sha(start_sha: str, llvm_symlink_dir: str) -> str:
|
|
if start_sha == "llvm":
|
|
return parse_ebuild_for_assignment(llvm_symlink_dir, "LLVM_HASH")
|
|
|
|
if start_sha == "llvm-next":
|
|
return parse_ebuild_for_assignment(llvm_symlink_dir, "LLVM_NEXT_HASH")
|
|
|
|
return start_sha
|
|
|
|
|
|
def find_patches_and_make_cl(
|
|
chroot_path: str,
|
|
patches: t.List[str],
|
|
start_rev: git_llvm_rev.Rev,
|
|
llvm_config: git_llvm_rev.LLVMConfig,
|
|
llvm_symlink_dir: str,
|
|
create_cl: bool,
|
|
skip_dependencies: bool,
|
|
reviewers: t.Optional[t.List[str]],
|
|
cc: t.Optional[t.List[str]],
|
|
platforms: t.List[str],
|
|
):
|
|
|
|
converted_patches = [
|
|
_convert_patch(llvm_config, skip_dependencies, p) for p in patches
|
|
]
|
|
potential_duplicates = _get_duplicate_shas(converted_patches)
|
|
if potential_duplicates:
|
|
err_msg = "\n".join(
|
|
f"{a.patch} == {b.patch}" for a, b in potential_duplicates
|
|
)
|
|
raise RuntimeError(f"Found Duplicate SHAs:\n{err_msg}")
|
|
|
|
# CL Related variables, only used if `create_cl`
|
|
symlinks_to_uprev = []
|
|
commit_messages = [
|
|
"llvm: get patches from upstream\n",
|
|
]
|
|
branch = f'get-upstream-{datetime.now().strftime("%Y%m%d%H%M%S%f")}'
|
|
|
|
if create_cl:
|
|
git.CreateBranch(llvm_symlink_dir, branch)
|
|
|
|
for parsed_patch in converted_patches:
|
|
# Find out the llvm projects changed in this commit
|
|
packages = get_package_names(parsed_patch.sha, llvm_config.dir)
|
|
# Find out the ebuild symlinks of the corresponding ChromeOS packages
|
|
symlinks = chroot.GetChrootEbuildPaths(
|
|
chroot_path,
|
|
[
|
|
"sys-devel/llvm" if package == "llvm" else "sys-libs/" + package
|
|
for package in packages
|
|
],
|
|
)
|
|
symlinks = chroot.ConvertChrootPathsToAbsolutePaths(
|
|
chroot_path, symlinks
|
|
)
|
|
# Create a local patch for all the affected llvm projects
|
|
create_patch_for_packages(
|
|
packages,
|
|
symlinks,
|
|
start_rev,
|
|
parsed_patch.rev,
|
|
parsed_patch.sha,
|
|
llvm_config.dir,
|
|
platforms=platforms,
|
|
)
|
|
if create_cl:
|
|
symlinks_to_uprev.extend(symlinks)
|
|
|
|
commit_messages.extend(
|
|
[
|
|
parsed_patch.git_msg(),
|
|
subprocess.check_output(
|
|
["git", "log", "-n1", "--oneline", parsed_patch.sha],
|
|
cwd=llvm_config.dir,
|
|
encoding="utf-8",
|
|
),
|
|
]
|
|
)
|
|
|
|
if parsed_patch.is_differential:
|
|
subprocess.check_output(
|
|
["git", "reset", "--hard", "HEAD^"], cwd=llvm_config.dir
|
|
)
|
|
|
|
if create_cl:
|
|
make_cl(
|
|
symlinks_to_uprev,
|
|
llvm_symlink_dir,
|
|
branch,
|
|
commit_messages,
|
|
reviewers,
|
|
cc,
|
|
)
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class ParsedPatch:
|
|
"""Class to keep track of bundled patch info."""
|
|
|
|
patch: str
|
|
sha: str
|
|
is_differential: bool
|
|
rev: t.Union[git_llvm_rev.Rev, str]
|
|
|
|
def git_msg(self) -> str:
|
|
if self.is_differential:
|
|
return f"\n\nreviews.llvm.org/{self.patch}\n"
|
|
return f"\n\nreviews.llvm.org/rG{self.sha}\n"
|
|
|
|
|
|
def _convert_patch(
|
|
llvm_config: git_llvm_rev.LLVMConfig, skip_dependencies: bool, patch: str
|
|
) -> ParsedPatch:
|
|
"""Extract git revision info from a patch.
|
|
|
|
Args:
|
|
llvm_config: LLVM configuration object.
|
|
skip_dependencies: Pass --skip-dependecies for to `arc`
|
|
patch: A single patch referent string.
|
|
|
|
Returns:
|
|
A [ParsedPatch] object.
|
|
"""
|
|
|
|
# git hash should only have lower-case letters
|
|
is_differential = patch.startswith("D")
|
|
if is_differential:
|
|
subprocess.check_output(
|
|
[
|
|
"arc",
|
|
"patch",
|
|
"--nobranch",
|
|
"--skip-dependencies" if skip_dependencies else "--revision",
|
|
patch,
|
|
],
|
|
cwd=llvm_config.dir,
|
|
)
|
|
sha = resolve_llvm_ref(llvm_config.dir, "HEAD")
|
|
rev = patch
|
|
else:
|
|
sha = resolve_llvm_ref(llvm_config.dir, patch)
|
|
rev = git_llvm_rev.translate_sha_to_rev(llvm_config, sha)
|
|
return ParsedPatch(
|
|
patch=patch, sha=sha, rev=rev, is_differential=is_differential
|
|
)
|
|
|
|
|
|
def _get_duplicate_shas(
|
|
patches: t.List[ParsedPatch],
|
|
) -> t.List[t.Tuple[ParsedPatch, ParsedPatch]]:
|
|
"""Return a list of Patches which have duplicate SHA's"""
|
|
return [
|
|
(left, right)
|
|
for i, left in enumerate(patches)
|
|
for right in patches[i + 1 :]
|
|
if left.sha == right.sha
|
|
]
|
|
|
|
|
|
def get_from_upstream(
|
|
chroot_path: str,
|
|
create_cl: bool,
|
|
start_sha: str,
|
|
patches: t.List[str],
|
|
platforms: t.List[str],
|
|
skip_dependencies: bool = False,
|
|
reviewers: t.List[str] = None,
|
|
cc: t.List[str] = None,
|
|
):
|
|
llvm_symlink = chroot.ConvertChrootPathsToAbsolutePaths(
|
|
chroot_path,
|
|
chroot.GetChrootEbuildPaths(chroot_path, ["sys-devel/llvm"]),
|
|
)[0]
|
|
llvm_symlink_dir = os.path.dirname(llvm_symlink)
|
|
|
|
git_status = subprocess.check_output(
|
|
["git", "status", "-s"], cwd=llvm_symlink_dir, encoding="utf-8"
|
|
)
|
|
|
|
if git_status:
|
|
error_path = os.path.dirname(os.path.dirname(llvm_symlink_dir))
|
|
raise ValueError(f"Uncommited changes detected in {error_path}")
|
|
|
|
start_sha = resolve_symbolic_sha(start_sha, llvm_symlink_dir)
|
|
logging.info("Base llvm hash == %s", start_sha)
|
|
|
|
llvm_config = git_llvm_rev.LLVMConfig(
|
|
remote="origin", dir=get_llvm_hash.GetAndUpdateLLVMProjectInLLVMTools()
|
|
)
|
|
start_sha = resolve_llvm_ref(llvm_config.dir, start_sha)
|
|
|
|
find_patches_and_make_cl(
|
|
chroot_path=chroot_path,
|
|
patches=patches,
|
|
platforms=platforms,
|
|
start_rev=git_llvm_rev.translate_sha_to_rev(llvm_config, start_sha),
|
|
llvm_config=llvm_config,
|
|
llvm_symlink_dir=llvm_symlink_dir,
|
|
create_cl=create_cl,
|
|
skip_dependencies=skip_dependencies,
|
|
reviewers=reviewers,
|
|
cc=cc,
|
|
)
|
|
logging.info("Complete.")
|
|
|
|
|
|
def main():
|
|
chroot.VerifyOutsideChroot()
|
|
logging.basicConfig(
|
|
format="%(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: %(message)s",
|
|
level=logging.INFO,
|
|
)
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog=__DOC_EPILOGUE,
|
|
)
|
|
parser.add_argument(
|
|
"--chroot_path",
|
|
default=os.path.join(os.path.expanduser("~"), "chromiumos"),
|
|
help="the path to the chroot (default: %(default)s)",
|
|
)
|
|
parser.add_argument(
|
|
"--start_sha",
|
|
default="llvm-next",
|
|
help="LLVM SHA that the patch should start applying at. You can specify "
|
|
'"llvm" or "llvm-next", as well. Defaults to %(default)s.',
|
|
)
|
|
parser.add_argument(
|
|
"--sha",
|
|
action="append",
|
|
default=[],
|
|
help="The LLVM git SHA to cherry-pick.",
|
|
)
|
|
parser.add_argument(
|
|
"--differential",
|
|
action="append",
|
|
default=[],
|
|
help="The LLVM differential revision to apply. Example: D1234."
|
|
" Cannot be used for changes already merged upstream; use --sha"
|
|
" instead for those.",
|
|
)
|
|
parser.add_argument(
|
|
"--platform",
|
|
action="append",
|
|
required=True,
|
|
help="Apply this patch to the give platform. Common options include "
|
|
'"chromiumos" and "android". Can be specified multiple times to '
|
|
"apply to multiple platforms",
|
|
)
|
|
parser.add_argument(
|
|
"--create_cl",
|
|
action="store_true",
|
|
help="Automatically create a CL if specified",
|
|
)
|
|
parser.add_argument(
|
|
"--skip_dependencies",
|
|
action="store_true",
|
|
help="Skips a LLVM differential revision's dependencies. Only valid "
|
|
"when --differential appears exactly once.",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
if not (args.sha or args.differential):
|
|
parser.error("--sha or --differential required")
|
|
|
|
if args.skip_dependencies and len(args.differential) != 1:
|
|
parser.error(
|
|
"--skip_dependencies is only valid when there's exactly one "
|
|
"supplied differential"
|
|
)
|
|
|
|
get_from_upstream(
|
|
chroot_path=args.chroot_path,
|
|
create_cl=args.create_cl,
|
|
start_sha=args.start_sha,
|
|
patches=args.sha + args.differential,
|
|
skip_dependencies=args.skip_dependencies,
|
|
platforms=args.platform,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|