You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
586 lines
17 KiB
586 lines
17 KiB
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
# Copyright 2019 The ChromiumOS Authors
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
# pylint: disable=protected-access
|
|
|
|
"""Tests for LLVM bisection."""
|
|
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import unittest
|
|
import unittest.mock as mock
|
|
|
|
import chroot
|
|
import get_llvm_hash
|
|
import git_llvm_rev
|
|
import llvm_bisection
|
|
import modify_a_tryjob
|
|
import test_helpers
|
|
|
|
|
|
class LLVMBisectionTest(unittest.TestCase):
|
|
"""Unittests for LLVM bisection."""
|
|
|
|
def testGetRemainingRangePassed(self):
|
|
start = 100
|
|
end = 150
|
|
|
|
test_tryjobs = [
|
|
{
|
|
"rev": 110,
|
|
"status": "good",
|
|
"link": "https://some_tryjob_1_url.com",
|
|
},
|
|
{
|
|
"rev": 120,
|
|
"status": "good",
|
|
"link": "https://some_tryjob_2_url.com",
|
|
},
|
|
{
|
|
"rev": 130,
|
|
"status": "pending",
|
|
"link": "https://some_tryjob_3_url.com",
|
|
},
|
|
{
|
|
"rev": 135,
|
|
"status": "skip",
|
|
"link": "https://some_tryjob_4_url.com",
|
|
},
|
|
{
|
|
"rev": 140,
|
|
"status": "bad",
|
|
"link": "https://some_tryjob_5_url.com",
|
|
},
|
|
]
|
|
|
|
# Tuple consists of the new good revision, the new bad revision, a set of
|
|
# 'pending' revisions, and a set of 'skip' revisions.
|
|
expected_revisions_tuple = 120, 140, {130}, {135}
|
|
|
|
self.assertEqual(
|
|
llvm_bisection.GetRemainingRange(start, end, test_tryjobs),
|
|
expected_revisions_tuple,
|
|
)
|
|
|
|
def testGetRemainingRangeFailedWithMissingStatus(self):
|
|
start = 100
|
|
end = 150
|
|
|
|
test_tryjobs = [
|
|
{
|
|
"rev": 105,
|
|
"status": "good",
|
|
"link": "https://some_tryjob_1_url.com",
|
|
},
|
|
{
|
|
"rev": 120,
|
|
"status": None,
|
|
"link": "https://some_tryjob_2_url.com",
|
|
},
|
|
{
|
|
"rev": 140,
|
|
"status": "bad",
|
|
"link": "https://some_tryjob_3_url.com",
|
|
},
|
|
]
|
|
|
|
with self.assertRaises(ValueError) as err:
|
|
llvm_bisection.GetRemainingRange(start, end, test_tryjobs)
|
|
|
|
error_message = (
|
|
'"status" is missing or has no value, please '
|
|
"go to %s and update it" % test_tryjobs[1]["link"]
|
|
)
|
|
self.assertEqual(str(err.exception), error_message)
|
|
|
|
def testGetRemainingRangeFailedWithInvalidRange(self):
|
|
start = 100
|
|
end = 150
|
|
|
|
test_tryjobs = [
|
|
{
|
|
"rev": 110,
|
|
"status": "bad",
|
|
"link": "https://some_tryjob_1_url.com",
|
|
},
|
|
{
|
|
"rev": 125,
|
|
"status": "skip",
|
|
"link": "https://some_tryjob_2_url.com",
|
|
},
|
|
{
|
|
"rev": 140,
|
|
"status": "good",
|
|
"link": "https://some_tryjob_3_url.com",
|
|
},
|
|
]
|
|
|
|
with self.assertRaises(AssertionError) as err:
|
|
llvm_bisection.GetRemainingRange(start, end, test_tryjobs)
|
|
|
|
expected_error_message = (
|
|
"Bisection is broken because %d (good) is >= "
|
|
"%d (bad)" % (test_tryjobs[2]["rev"], test_tryjobs[0]["rev"])
|
|
)
|
|
|
|
self.assertEqual(str(err.exception), expected_error_message)
|
|
|
|
@mock.patch.object(get_llvm_hash, "GetGitHashFrom")
|
|
def testGetCommitsBetweenPassed(self, mock_get_git_hash):
|
|
start = git_llvm_rev.base_llvm_revision
|
|
end = start + 10
|
|
test_pending_revisions = {start + 7}
|
|
test_skip_revisions = {
|
|
start + 1,
|
|
start + 2,
|
|
start + 4,
|
|
start + 8,
|
|
start + 9,
|
|
}
|
|
parallel = 3
|
|
abs_path_to_src = "/abs/path/to/src"
|
|
|
|
revs = ["a123testhash3", "a123testhash5"]
|
|
mock_get_git_hash.side_effect = revs
|
|
|
|
git_hashes = [
|
|
git_llvm_rev.base_llvm_revision + 3,
|
|
git_llvm_rev.base_llvm_revision + 5,
|
|
]
|
|
|
|
self.assertEqual(
|
|
llvm_bisection.GetCommitsBetween(
|
|
start,
|
|
end,
|
|
parallel,
|
|
abs_path_to_src,
|
|
test_pending_revisions,
|
|
test_skip_revisions,
|
|
),
|
|
(git_hashes, revs),
|
|
)
|
|
|
|
def testLoadStatusFilePassedWithExistingFile(self):
|
|
start = 100
|
|
end = 150
|
|
|
|
test_bisect_state = {"start": start, "end": end, "jobs": []}
|
|
|
|
# Simulate that the status file exists.
|
|
with test_helpers.CreateTemporaryJsonFile() as temp_json_file:
|
|
with open(temp_json_file, "w") as f:
|
|
test_helpers.WritePrettyJsonFile(test_bisect_state, f)
|
|
|
|
self.assertEqual(
|
|
llvm_bisection.LoadStatusFile(temp_json_file, start, end),
|
|
test_bisect_state,
|
|
)
|
|
|
|
def testLoadStatusFilePassedWithoutExistingFile(self):
|
|
start = 200
|
|
end = 250
|
|
|
|
expected_bisect_state = {"start": start, "end": end, "jobs": []}
|
|
|
|
last_tested = "/abs/path/to/file_that_does_not_exist.json"
|
|
|
|
self.assertEqual(
|
|
llvm_bisection.LoadStatusFile(last_tested, start, end),
|
|
expected_bisect_state,
|
|
)
|
|
|
|
@mock.patch.object(modify_a_tryjob, "AddTryjob")
|
|
def testBisectPassed(self, mock_add_tryjob):
|
|
|
|
git_hash_list = ["a123testhash1", "a123testhash2", "a123testhash3"]
|
|
revisions_list = [102, 104, 106]
|
|
|
|
# Simulate behavior of `AddTryjob()` when successfully launched a tryjob for
|
|
# the updated packages.
|
|
@test_helpers.CallCountsToMockFunctions
|
|
def MockAddTryjob(
|
|
call_count,
|
|
_packages,
|
|
_git_hash,
|
|
_revision,
|
|
_chroot_path,
|
|
_patch_file,
|
|
_extra_cls,
|
|
_options,
|
|
_builder,
|
|
_verbose,
|
|
_svn_revision,
|
|
):
|
|
|
|
if call_count < 2:
|
|
return {"rev": revisions_list[call_count], "status": "pending"}
|
|
|
|
# Simulate an exception happened along the way when updating the
|
|
# packages' `LLVM_NEXT_HASH`.
|
|
if call_count == 2:
|
|
raise ValueError("Unable to launch tryjob")
|
|
|
|
assert False, "Called `AddTryjob()` more than expected."
|
|
|
|
# Use the test function to simulate `AddTryjob()`.
|
|
mock_add_tryjob.side_effect = MockAddTryjob
|
|
|
|
start = 100
|
|
end = 110
|
|
|
|
bisection_contents = {"start": start, "end": end, "jobs": []}
|
|
|
|
args_output = test_helpers.ArgsOutputTest()
|
|
|
|
packages = ["sys-devel/llvm"]
|
|
patch_file = "/abs/path/to/PATCHES.json"
|
|
|
|
# Create a temporary .JSON file to simulate a status file for bisection.
|
|
with test_helpers.CreateTemporaryJsonFile() as temp_json_file:
|
|
with open(temp_json_file, "w") as f:
|
|
test_helpers.WritePrettyJsonFile(bisection_contents, f)
|
|
|
|
# Verify that the status file is updated when an exception happened when
|
|
# attempting to launch a revision (i.e. progress is not lost).
|
|
with self.assertRaises(ValueError) as err:
|
|
llvm_bisection.Bisect(
|
|
revisions_list,
|
|
git_hash_list,
|
|
bisection_contents,
|
|
temp_json_file,
|
|
packages,
|
|
args_output.chroot_path,
|
|
patch_file,
|
|
args_output.extra_change_lists,
|
|
args_output.options,
|
|
args_output.builders,
|
|
args_output.verbose,
|
|
)
|
|
|
|
expected_bisection_contents = {
|
|
"start": start,
|
|
"end": end,
|
|
"jobs": [
|
|
{"rev": revisions_list[0], "status": "pending"},
|
|
{"rev": revisions_list[1], "status": "pending"},
|
|
],
|
|
}
|
|
|
|
# Verify that the launched tryjobs were added to the status file when
|
|
# an exception happened.
|
|
with open(temp_json_file) as f:
|
|
json_contents = json.load(f)
|
|
|
|
self.assertEqual(json_contents, expected_bisection_contents)
|
|
|
|
self.assertEqual(str(err.exception), "Unable to launch tryjob")
|
|
|
|
self.assertEqual(mock_add_tryjob.call_count, 3)
|
|
|
|
@mock.patch.object(subprocess, "check_output", return_value=None)
|
|
@mock.patch.object(
|
|
get_llvm_hash.LLVMHash, "GetLLVMHash", return_value="a123testhash4"
|
|
)
|
|
@mock.patch.object(llvm_bisection, "GetCommitsBetween")
|
|
@mock.patch.object(llvm_bisection, "GetRemainingRange")
|
|
@mock.patch.object(llvm_bisection, "LoadStatusFile")
|
|
@mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True)
|
|
def testMainPassed(
|
|
self,
|
|
mock_outside_chroot,
|
|
mock_load_status_file,
|
|
mock_get_range,
|
|
mock_get_revision_and_hash_list,
|
|
_mock_get_bad_llvm_hash,
|
|
mock_abandon_cl,
|
|
):
|
|
|
|
start = 500
|
|
end = 502
|
|
cl = 1
|
|
|
|
bisect_state = {
|
|
"start": start,
|
|
"end": end,
|
|
"jobs": [{"rev": 501, "status": "bad", "cl": cl}],
|
|
}
|
|
|
|
skip_revisions = {501}
|
|
pending_revisions = {}
|
|
|
|
mock_load_status_file.return_value = bisect_state
|
|
|
|
mock_get_range.return_value = (
|
|
start,
|
|
end,
|
|
pending_revisions,
|
|
skip_revisions,
|
|
)
|
|
|
|
mock_get_revision_and_hash_list.return_value = [], []
|
|
|
|
args_output = test_helpers.ArgsOutputTest()
|
|
args_output.start_rev = start
|
|
args_output.end_rev = end
|
|
args_output.parallel = 3
|
|
args_output.src_path = None
|
|
args_output.chroot_path = "somepath"
|
|
args_output.cleanup = True
|
|
|
|
self.assertEqual(
|
|
llvm_bisection.main(args_output),
|
|
llvm_bisection.BisectionExitStatus.BISECTION_COMPLETE.value,
|
|
)
|
|
|
|
mock_outside_chroot.assert_called_once()
|
|
|
|
mock_load_status_file.assert_called_once()
|
|
|
|
mock_get_range.assert_called_once()
|
|
|
|
mock_get_revision_and_hash_list.assert_called_once()
|
|
|
|
mock_abandon_cl.assert_called_once()
|
|
self.assertEqual(
|
|
mock_abandon_cl.call_args,
|
|
mock.call(
|
|
[
|
|
os.path.join(
|
|
args_output.chroot_path, "chromite/bin/gerrit"
|
|
),
|
|
"abandon",
|
|
str(cl),
|
|
],
|
|
stderr=subprocess.STDOUT,
|
|
encoding="utf-8",
|
|
),
|
|
)
|
|
|
|
@mock.patch.object(llvm_bisection, "LoadStatusFile")
|
|
@mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True)
|
|
def testMainFailedWithInvalidRange(
|
|
self, mock_outside_chroot, mock_load_status_file
|
|
):
|
|
|
|
start = 500
|
|
end = 502
|
|
|
|
bisect_state = {
|
|
"start": start - 1,
|
|
"end": end,
|
|
}
|
|
|
|
mock_load_status_file.return_value = bisect_state
|
|
|
|
args_output = test_helpers.ArgsOutputTest()
|
|
args_output.start_rev = start
|
|
args_output.end_rev = end
|
|
args_output.parallel = 3
|
|
args_output.src_path = None
|
|
|
|
with self.assertRaises(ValueError) as err:
|
|
llvm_bisection.main(args_output)
|
|
|
|
error_message = (
|
|
f"The start {start} or the end {end} version provided is "
|
|
f'different than "start" {bisect_state["start"]} or "end" '
|
|
f'{bisect_state["end"]} in the .JSON file'
|
|
)
|
|
|
|
self.assertEqual(str(err.exception), error_message)
|
|
|
|
mock_outside_chroot.assert_called_once()
|
|
|
|
mock_load_status_file.assert_called_once()
|
|
|
|
@mock.patch.object(llvm_bisection, "GetCommitsBetween")
|
|
@mock.patch.object(llvm_bisection, "GetRemainingRange")
|
|
@mock.patch.object(llvm_bisection, "LoadStatusFile")
|
|
@mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True)
|
|
def testMainFailedWithPendingBuilds(
|
|
self,
|
|
mock_outside_chroot,
|
|
mock_load_status_file,
|
|
mock_get_range,
|
|
mock_get_revision_and_hash_list,
|
|
):
|
|
|
|
start = 500
|
|
end = 502
|
|
rev = 501
|
|
|
|
bisect_state = {
|
|
"start": start,
|
|
"end": end,
|
|
"jobs": [{"rev": rev, "status": "pending"}],
|
|
}
|
|
|
|
skip_revisions = {}
|
|
pending_revisions = {rev}
|
|
|
|
mock_load_status_file.return_value = bisect_state
|
|
|
|
mock_get_range.return_value = (
|
|
start,
|
|
end,
|
|
pending_revisions,
|
|
skip_revisions,
|
|
)
|
|
|
|
mock_get_revision_and_hash_list.return_value = [], []
|
|
|
|
args_output = test_helpers.ArgsOutputTest()
|
|
args_output.start_rev = start
|
|
args_output.end_rev = end
|
|
args_output.parallel = 3
|
|
args_output.src_path = None
|
|
|
|
with self.assertRaises(ValueError) as err:
|
|
llvm_bisection.main(args_output)
|
|
|
|
error_message = (
|
|
f"No revisions between start {start} and end {end} to "
|
|
"create tryjobs\nThe following tryjobs are pending:\n"
|
|
f"{rev}\n"
|
|
)
|
|
|
|
self.assertEqual(str(err.exception), error_message)
|
|
|
|
mock_outside_chroot.assert_called_once()
|
|
|
|
mock_load_status_file.assert_called_once()
|
|
|
|
mock_get_range.assert_called_once()
|
|
|
|
mock_get_revision_and_hash_list.assert_called_once()
|
|
|
|
@mock.patch.object(llvm_bisection, "GetCommitsBetween")
|
|
@mock.patch.object(llvm_bisection, "GetRemainingRange")
|
|
@mock.patch.object(llvm_bisection, "LoadStatusFile")
|
|
@mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True)
|
|
def testMainFailedWithDuplicateBuilds(
|
|
self,
|
|
mock_outside_chroot,
|
|
mock_load_status_file,
|
|
mock_get_range,
|
|
mock_get_revision_and_hash_list,
|
|
):
|
|
|
|
start = 500
|
|
end = 502
|
|
rev = 501
|
|
git_hash = "a123testhash1"
|
|
|
|
bisect_state = {
|
|
"start": start,
|
|
"end": end,
|
|
"jobs": [{"rev": rev, "status": "pending"}],
|
|
}
|
|
|
|
skip_revisions = {}
|
|
pending_revisions = {rev}
|
|
|
|
mock_load_status_file.return_value = bisect_state
|
|
|
|
mock_get_range.return_value = (
|
|
start,
|
|
end,
|
|
pending_revisions,
|
|
skip_revisions,
|
|
)
|
|
|
|
mock_get_revision_and_hash_list.return_value = [rev], [git_hash]
|
|
|
|
args_output = test_helpers.ArgsOutputTest()
|
|
args_output.start_rev = start
|
|
args_output.end_rev = end
|
|
args_output.parallel = 3
|
|
args_output.src_path = None
|
|
|
|
with self.assertRaises(ValueError) as err:
|
|
llvm_bisection.main(args_output)
|
|
|
|
error_message = 'Revision %d exists already in "jobs"' % rev
|
|
self.assertEqual(str(err.exception), error_message)
|
|
|
|
mock_outside_chroot.assert_called_once()
|
|
|
|
mock_load_status_file.assert_called_once()
|
|
|
|
mock_get_range.assert_called_once()
|
|
|
|
mock_get_revision_and_hash_list.assert_called_once()
|
|
|
|
@mock.patch.object(subprocess, "check_output", return_value=None)
|
|
@mock.patch.object(
|
|
get_llvm_hash.LLVMHash, "GetLLVMHash", return_value="a123testhash4"
|
|
)
|
|
@mock.patch.object(llvm_bisection, "GetCommitsBetween")
|
|
@mock.patch.object(llvm_bisection, "GetRemainingRange")
|
|
@mock.patch.object(llvm_bisection, "LoadStatusFile")
|
|
@mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True)
|
|
def testMainFailedToAbandonCL(
|
|
self,
|
|
mock_outside_chroot,
|
|
mock_load_status_file,
|
|
mock_get_range,
|
|
mock_get_revision_and_hash_list,
|
|
_mock_get_bad_llvm_hash,
|
|
mock_abandon_cl,
|
|
):
|
|
|
|
start = 500
|
|
end = 502
|
|
|
|
bisect_state = {
|
|
"start": start,
|
|
"end": end,
|
|
"jobs": [{"rev": 501, "status": "bad", "cl": 0}],
|
|
}
|
|
|
|
skip_revisions = {501}
|
|
pending_revisions = {}
|
|
|
|
mock_load_status_file.return_value = bisect_state
|
|
|
|
mock_get_range.return_value = (
|
|
start,
|
|
end,
|
|
pending_revisions,
|
|
skip_revisions,
|
|
)
|
|
|
|
mock_get_revision_and_hash_list.return_value = ([], [])
|
|
|
|
error_message = "Error message."
|
|
mock_abandon_cl.side_effect = subprocess.CalledProcessError(
|
|
returncode=1, cmd=[], output=error_message
|
|
)
|
|
|
|
args_output = test_helpers.ArgsOutputTest()
|
|
args_output.start_rev = start
|
|
args_output.end_rev = end
|
|
args_output.parallel = 3
|
|
args_output.src_path = None
|
|
args_output.cleanup = True
|
|
|
|
with self.assertRaises(subprocess.CalledProcessError) as err:
|
|
llvm_bisection.main(args_output)
|
|
|
|
self.assertEqual(err.exception.output, error_message)
|
|
|
|
mock_outside_chroot.assert_called_once()
|
|
|
|
mock_load_status_file.assert_called_once()
|
|
|
|
mock_get_range.assert_called_once()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|