You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
546 lines
17 KiB
546 lines
17 KiB
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
|
|
# Copyright 2013 The ChromiumOS Authors
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
"""Testing of benchmark_run."""
|
|
|
|
|
|
import inspect
|
|
import unittest
|
|
import unittest.mock as mock
|
|
|
|
from benchmark import Benchmark
|
|
import benchmark_run
|
|
from cros_utils import logger
|
|
from label import MockLabel
|
|
from machine_manager import MachineManager
|
|
from machine_manager import MockCrosMachine
|
|
from machine_manager import MockMachineManager
|
|
from results_cache import CacheConditions
|
|
from results_cache import MockResultsCache
|
|
from results_cache import Result
|
|
from results_cache import ResultsCache
|
|
from suite_runner import MockSuiteRunner
|
|
from suite_runner import SuiteRunner
|
|
|
|
|
|
class BenchmarkRunTest(unittest.TestCase):
|
|
"""Unit tests for the BenchmarkRun class and all of its methods."""
|
|
|
|
def setUp(self):
|
|
self.status = []
|
|
self.called_ReadCache = None
|
|
self.log_error = []
|
|
self.log_output = []
|
|
self.err_msg = None
|
|
self.test_benchmark = Benchmark(
|
|
"page_cycler.netsim.top_10", # name
|
|
"page_cycler.netsim.top_10", # test_name
|
|
"", # test_args
|
|
1, # iterations
|
|
False, # rm_chroot_tmp
|
|
"", # perf_args
|
|
suite="telemetry_Crosperf",
|
|
) # suite
|
|
|
|
self.test_label = MockLabel(
|
|
"test1",
|
|
"build",
|
|
"image1",
|
|
"autotest_dir",
|
|
"debug_dir",
|
|
"/tmp/test_benchmark_run",
|
|
"x86-alex",
|
|
"chromeos2-row1-rack4-host9.cros",
|
|
image_args="",
|
|
cache_dir="",
|
|
cache_only=False,
|
|
log_level="average",
|
|
compiler="gcc",
|
|
crosfleet=False,
|
|
)
|
|
|
|
self.test_cache_conditions = [
|
|
CacheConditions.CACHE_FILE_EXISTS,
|
|
CacheConditions.CHECKSUMS_MATCH,
|
|
]
|
|
|
|
self.mock_logger = logger.GetLogger(log_dir="", mock=True)
|
|
|
|
self.mock_machine_manager = mock.Mock(spec=MachineManager)
|
|
|
|
def testDryRun(self):
|
|
my_label = MockLabel(
|
|
"test1",
|
|
"build",
|
|
"image1",
|
|
"autotest_dir",
|
|
"debug_dir",
|
|
"/tmp/test_benchmark_run",
|
|
"x86-alex",
|
|
"chromeos2-row1-rack4-host9.cros",
|
|
image_args="",
|
|
cache_dir="",
|
|
cache_only=False,
|
|
log_level="average",
|
|
compiler="gcc",
|
|
crosfleet=False,
|
|
)
|
|
|
|
logging_level = "average"
|
|
m = MockMachineManager("/tmp/chromeos_root", 0, logging_level, "")
|
|
m.AddMachine("chromeos2-row1-rack4-host9.cros")
|
|
bench = Benchmark(
|
|
"page_cycler.netsim.top_10", # name
|
|
"page_cycler.netsim.top_10", # test_name
|
|
"", # test_args
|
|
1, # iterations
|
|
False, # rm_chroot_tmp
|
|
"", # perf_args
|
|
suite="telemetry_Crosperf",
|
|
) # suite
|
|
dut_conf = {
|
|
"cooldown_time": 5,
|
|
"cooldown_temp": 45,
|
|
"governor": "powersave",
|
|
"cpu_usage": "big_only",
|
|
"cpu_freq_pct": 80,
|
|
}
|
|
b = benchmark_run.MockBenchmarkRun(
|
|
"test run",
|
|
bench,
|
|
my_label,
|
|
1,
|
|
[],
|
|
m,
|
|
logger.GetLogger(),
|
|
logging_level,
|
|
"",
|
|
dut_conf,
|
|
)
|
|
b.cache = MockResultsCache()
|
|
b.suite_runner = MockSuiteRunner()
|
|
b.start()
|
|
|
|
# Make sure the arguments to BenchmarkRun.__init__ have not changed
|
|
# since the last time this test was updated:
|
|
args_list = [
|
|
"self",
|
|
"name",
|
|
"benchmark",
|
|
"label",
|
|
"iteration",
|
|
"cache_conditions",
|
|
"machine_manager",
|
|
"logger_to_use",
|
|
"log_level",
|
|
"share_cache",
|
|
"dut_config",
|
|
]
|
|
arg_spec = inspect.getfullargspec(benchmark_run.BenchmarkRun.__init__)
|
|
self.assertEqual(len(arg_spec.args), len(args_list))
|
|
self.assertEqual(arg_spec.args, args_list)
|
|
|
|
def test_init(self):
|
|
# Nothing really worth testing here; just field assignments.
|
|
pass
|
|
|
|
def test_read_cache(self):
|
|
# Nothing really worth testing here, either.
|
|
pass
|
|
|
|
def test_run(self):
|
|
br = benchmark_run.BenchmarkRun(
|
|
"test_run",
|
|
self.test_benchmark,
|
|
self.test_label,
|
|
1,
|
|
self.test_cache_conditions,
|
|
self.mock_machine_manager,
|
|
self.mock_logger,
|
|
"average",
|
|
"",
|
|
{},
|
|
)
|
|
|
|
def MockLogOutput(msg, print_to_console=False):
|
|
"""Helper function for test_run."""
|
|
del print_to_console
|
|
self.log_output.append(msg)
|
|
|
|
def MockLogError(msg, print_to_console=False):
|
|
"""Helper function for test_run."""
|
|
del print_to_console
|
|
self.log_error.append(msg)
|
|
|
|
def MockRecordStatus(msg):
|
|
"""Helper function for test_run."""
|
|
self.status.append(msg)
|
|
|
|
def FakeReadCache():
|
|
"""Helper function for test_run."""
|
|
br.cache = mock.Mock(spec=ResultsCache)
|
|
self.called_ReadCache = True
|
|
return 0
|
|
|
|
def FakeReadCacheSucceed():
|
|
"""Helper function for test_run."""
|
|
br.cache = mock.Mock(spec=ResultsCache)
|
|
br.result = mock.Mock(spec=Result)
|
|
br.result.out = "result.out stuff"
|
|
br.result.err = "result.err stuff"
|
|
br.result.retval = 0
|
|
self.called_ReadCache = True
|
|
return 0
|
|
|
|
def FakeReadCacheException():
|
|
"""Helper function for test_run."""
|
|
raise RuntimeError(
|
|
"This is an exception test; it is supposed to happen"
|
|
)
|
|
|
|
def FakeAcquireMachine():
|
|
"""Helper function for test_run."""
|
|
mock_machine = MockCrosMachine(
|
|
"chromeos1-row3-rack5-host7.cros", "chromeos", "average"
|
|
)
|
|
return mock_machine
|
|
|
|
def FakeRunTest(_machine):
|
|
"""Helper function for test_run."""
|
|
mock_result = mock.Mock(spec=Result)
|
|
mock_result.retval = 0
|
|
return mock_result
|
|
|
|
def FakeRunTestFail(_machine):
|
|
"""Helper function for test_run."""
|
|
mock_result = mock.Mock(spec=Result)
|
|
mock_result.retval = 1
|
|
return mock_result
|
|
|
|
def ResetTestValues():
|
|
"""Helper function for test_run."""
|
|
self.log_output = []
|
|
self.log_error = []
|
|
self.status = []
|
|
br.result = None
|
|
self.called_ReadCache = False
|
|
|
|
# Assign all the fake functions to the appropriate objects.
|
|
br.logger().LogOutput = MockLogOutput
|
|
br.logger().LogError = MockLogError
|
|
br.timeline.Record = MockRecordStatus
|
|
br.ReadCache = FakeReadCache
|
|
br.RunTest = FakeRunTest
|
|
br.AcquireMachine = FakeAcquireMachine
|
|
|
|
# First test: No cache hit, all goes well.
|
|
ResetTestValues()
|
|
br.run()
|
|
self.assertTrue(self.called_ReadCache)
|
|
self.assertEqual(
|
|
self.log_output,
|
|
[
|
|
"test_run: No cache hit.",
|
|
"Releasing machine: chromeos1-row3-rack5-host7.cros",
|
|
"Released machine: chromeos1-row3-rack5-host7.cros",
|
|
],
|
|
)
|
|
self.assertEqual(len(self.log_error), 0)
|
|
self.assertEqual(self.status, ["WAITING", "SUCCEEDED"])
|
|
|
|
# Second test: No cached result found; test run was "terminated" for some
|
|
# reason.
|
|
ResetTestValues()
|
|
br.terminated = True
|
|
br.run()
|
|
self.assertTrue(self.called_ReadCache)
|
|
self.assertEqual(
|
|
self.log_output,
|
|
[
|
|
"test_run: No cache hit.",
|
|
"Releasing machine: chromeos1-row3-rack5-host7.cros",
|
|
"Released machine: chromeos1-row3-rack5-host7.cros",
|
|
],
|
|
)
|
|
self.assertEqual(len(self.log_error), 0)
|
|
self.assertEqual(self.status, ["WAITING"])
|
|
|
|
# Third test. No cached result found; RunTest failed for some reason.
|
|
ResetTestValues()
|
|
br.terminated = False
|
|
br.RunTest = FakeRunTestFail
|
|
br.run()
|
|
self.assertTrue(self.called_ReadCache)
|
|
self.assertEqual(
|
|
self.log_output,
|
|
[
|
|
"test_run: No cache hit.",
|
|
"Releasing machine: chromeos1-row3-rack5-host7.cros",
|
|
"Released machine: chromeos1-row3-rack5-host7.cros",
|
|
],
|
|
)
|
|
self.assertEqual(len(self.log_error), 0)
|
|
self.assertEqual(self.status, ["WAITING", "FAILED"])
|
|
|
|
# Fourth test: ReadCache found a cached result.
|
|
ResetTestValues()
|
|
br.RunTest = FakeRunTest
|
|
br.ReadCache = FakeReadCacheSucceed
|
|
br.run()
|
|
self.assertTrue(self.called_ReadCache)
|
|
self.assertEqual(
|
|
self.log_output,
|
|
[
|
|
"test_run: Cache hit.",
|
|
"result.out stuff",
|
|
"Releasing machine: chromeos1-row3-rack5-host7.cros",
|
|
"Released machine: chromeos1-row3-rack5-host7.cros",
|
|
],
|
|
)
|
|
self.assertEqual(self.log_error, ["result.err stuff"])
|
|
self.assertEqual(self.status, ["SUCCEEDED"])
|
|
|
|
# Fifth test: ReadCache generates an exception; does the try/finally block
|
|
# work?
|
|
ResetTestValues()
|
|
br.ReadCache = FakeReadCacheException
|
|
br.machine = FakeAcquireMachine()
|
|
br.run()
|
|
self.assertEqual(
|
|
self.log_error,
|
|
[
|
|
"Benchmark run: 'test_run' failed: This is an exception test; it is "
|
|
"supposed to happen"
|
|
],
|
|
)
|
|
self.assertEqual(self.status, ["FAILED"])
|
|
|
|
def test_terminate_pass(self):
|
|
br = benchmark_run.BenchmarkRun(
|
|
"test_run",
|
|
self.test_benchmark,
|
|
self.test_label,
|
|
1,
|
|
self.test_cache_conditions,
|
|
self.mock_machine_manager,
|
|
self.mock_logger,
|
|
"average",
|
|
"",
|
|
{},
|
|
)
|
|
|
|
def GetLastEventPassed():
|
|
"""Helper function for test_terminate_pass"""
|
|
return benchmark_run.STATUS_SUCCEEDED
|
|
|
|
def RecordStub(status):
|
|
"""Helper function for test_terminate_pass"""
|
|
self.status = status
|
|
|
|
self.status = benchmark_run.STATUS_SUCCEEDED
|
|
self.assertFalse(br.terminated)
|
|
self.assertFalse(br.suite_runner.CommandTerminator().IsTerminated())
|
|
|
|
br.timeline.GetLastEvent = GetLastEventPassed
|
|
br.timeline.Record = RecordStub
|
|
|
|
br.Terminate()
|
|
|
|
self.assertTrue(br.terminated)
|
|
self.assertTrue(br.suite_runner.CommandTerminator().IsTerminated())
|
|
self.assertEqual(self.status, benchmark_run.STATUS_FAILED)
|
|
|
|
def test_terminate_fail(self):
|
|
br = benchmark_run.BenchmarkRun(
|
|
"test_run",
|
|
self.test_benchmark,
|
|
self.test_label,
|
|
1,
|
|
self.test_cache_conditions,
|
|
self.mock_machine_manager,
|
|
self.mock_logger,
|
|
"average",
|
|
"",
|
|
{},
|
|
)
|
|
|
|
def GetLastEventFailed():
|
|
"""Helper function for test_terminate_fail"""
|
|
return benchmark_run.STATUS_FAILED
|
|
|
|
def RecordStub(status):
|
|
"""Helper function for test_terminate_fail"""
|
|
self.status = status
|
|
|
|
self.status = benchmark_run.STATUS_SUCCEEDED
|
|
self.assertFalse(br.terminated)
|
|
self.assertFalse(br.suite_runner.CommandTerminator().IsTerminated())
|
|
|
|
br.timeline.GetLastEvent = GetLastEventFailed
|
|
br.timeline.Record = RecordStub
|
|
|
|
br.Terminate()
|
|
|
|
self.assertTrue(br.terminated)
|
|
self.assertTrue(br.suite_runner.CommandTerminator().IsTerminated())
|
|
self.assertEqual(self.status, benchmark_run.STATUS_SUCCEEDED)
|
|
|
|
def test_acquire_machine(self):
|
|
br = benchmark_run.BenchmarkRun(
|
|
"test_run",
|
|
self.test_benchmark,
|
|
self.test_label,
|
|
1,
|
|
self.test_cache_conditions,
|
|
self.mock_machine_manager,
|
|
self.mock_logger,
|
|
"average",
|
|
"",
|
|
{},
|
|
)
|
|
|
|
br.terminated = True
|
|
self.assertRaises(Exception, br.AcquireMachine)
|
|
|
|
br.terminated = False
|
|
mock_machine = MockCrosMachine(
|
|
"chromeos1-row3-rack5-host7.cros", "chromeos", "average"
|
|
)
|
|
self.mock_machine_manager.AcquireMachine.return_value = mock_machine
|
|
|
|
machine = br.AcquireMachine()
|
|
self.assertEqual(machine.name, "chromeos1-row3-rack5-host7.cros")
|
|
|
|
def test_get_extra_autotest_args(self):
|
|
br = benchmark_run.BenchmarkRun(
|
|
"test_run",
|
|
self.test_benchmark,
|
|
self.test_label,
|
|
1,
|
|
self.test_cache_conditions,
|
|
self.mock_machine_manager,
|
|
self.mock_logger,
|
|
"average",
|
|
"",
|
|
{},
|
|
)
|
|
|
|
def MockLogError(err_msg):
|
|
"""Helper function for test_get_extra_autotest_args"""
|
|
self.err_msg = err_msg
|
|
|
|
self.mock_logger.LogError = MockLogError
|
|
|
|
result = br.GetExtraAutotestArgs()
|
|
self.assertEqual(result, "")
|
|
|
|
self.test_benchmark.perf_args = "record -e cycles"
|
|
result = br.GetExtraAutotestArgs()
|
|
self.assertEqual(
|
|
result,
|
|
"--profiler=custom_perf --profiler_args='perf_options=\"record -a -e "
|
|
"cycles\"'",
|
|
)
|
|
|
|
self.test_benchmark.perf_args = "record -e cycles"
|
|
self.test_benchmark.suite = "test_that"
|
|
result = br.GetExtraAutotestArgs()
|
|
self.assertEqual(result, "")
|
|
self.assertEqual(
|
|
self.err_msg, "Non-telemetry benchmark does not support profiler."
|
|
)
|
|
|
|
self.test_benchmark.perf_args = "junk args"
|
|
self.test_benchmark.suite = "telemetry_Crosperf"
|
|
self.assertRaises(Exception, br.GetExtraAutotestArgs)
|
|
|
|
@mock.patch.object(SuiteRunner, "Run")
|
|
@mock.patch.object(Result, "CreateFromRun")
|
|
def test_run_test(self, mock_result, mock_runner):
|
|
br = benchmark_run.BenchmarkRun(
|
|
"test_run",
|
|
self.test_benchmark,
|
|
self.test_label,
|
|
1,
|
|
self.test_cache_conditions,
|
|
self.mock_machine_manager,
|
|
self.mock_logger,
|
|
"average",
|
|
"",
|
|
{},
|
|
)
|
|
|
|
self.status = []
|
|
|
|
def MockRecord(status):
|
|
self.status.append(status)
|
|
|
|
br.timeline.Record = MockRecord
|
|
mock_machine = MockCrosMachine(
|
|
"chromeos1-row3-rack5-host7.cros", "chromeos", "average"
|
|
)
|
|
mock_runner.return_value = [0, "{'Score':100}", ""]
|
|
|
|
br.RunTest(mock_machine)
|
|
|
|
self.assertTrue(br.run_completed)
|
|
self.assertEqual(
|
|
self.status,
|
|
[benchmark_run.STATUS_IMAGING, benchmark_run.STATUS_RUNNING],
|
|
)
|
|
|
|
self.assertEqual(br.machine_manager.ImageMachine.call_count, 1)
|
|
br.machine_manager.ImageMachine.assert_called_with(
|
|
mock_machine, self.test_label
|
|
)
|
|
self.assertEqual(mock_runner.call_count, 1)
|
|
mock_runner.assert_called_with(
|
|
mock_machine, br.label, br.benchmark, "", br.profiler_args
|
|
)
|
|
|
|
self.assertEqual(mock_result.call_count, 1)
|
|
mock_result.assert_called_with(
|
|
self.mock_logger,
|
|
"average",
|
|
self.test_label,
|
|
None,
|
|
"{'Score':100}",
|
|
"",
|
|
0,
|
|
"page_cycler.netsim.top_10",
|
|
"telemetry_Crosperf",
|
|
"",
|
|
)
|
|
|
|
def test_set_cache_conditions(self):
|
|
br = benchmark_run.BenchmarkRun(
|
|
"test_run",
|
|
self.test_benchmark,
|
|
self.test_label,
|
|
1,
|
|
self.test_cache_conditions,
|
|
self.mock_machine_manager,
|
|
self.mock_logger,
|
|
"average",
|
|
"",
|
|
{},
|
|
)
|
|
|
|
phony_cache_conditions = [123, 456, True, False]
|
|
|
|
self.assertEqual(br.cache_conditions, self.test_cache_conditions)
|
|
|
|
br.SetCacheConditions(phony_cache_conditions)
|
|
self.assertEqual(br.cache_conditions, phony_cache_conditions)
|
|
|
|
br.SetCacheConditions(self.test_cache_conditions)
|
|
self.assertEqual(br.cache_conditions, self.test_cache_conditions)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|