You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
673 lines
24 KiB
673 lines
24 KiB
# -*- coding: utf-8 -*-
|
|
# Copyright 2013 The ChromiumOS Authors
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
"""A module to generate experiments."""
|
|
|
|
|
|
import os
|
|
import re
|
|
import socket
|
|
import sys
|
|
|
|
from benchmark import Benchmark
|
|
from cros_utils import command_executer
|
|
from cros_utils import logger
|
|
from experiment import Experiment
|
|
import file_lock_machine
|
|
from label import Label
|
|
from label import MockLabel
|
|
from results_cache import CacheConditions
|
|
import test_flag
|
|
|
|
import config
|
|
|
|
|
|
# Users may want to run Telemetry tests either individually, or in
|
|
# specified sets. Here we define sets of tests that users may want
|
|
# to run together.
|
|
|
|
telemetry_perfv2_tests = [
|
|
"kraken",
|
|
"octane",
|
|
]
|
|
|
|
telemetry_pagecycler_tests = [
|
|
"page_cycler_v2.intl_ar_fa_he",
|
|
"page_cycler_v2.intl_es_fr_pt-BR",
|
|
"page_cycler_v2.intl_hi_ru",
|
|
"page_cycler_v2.intl_ja_zh",
|
|
"page_cycler_v2.intl_ko_th_vi",
|
|
"page_cycler_v2.typical_25",
|
|
]
|
|
|
|
telemetry_toolchain_old_perf_tests = [
|
|
"page_cycler_v2.intl_es_fr_pt-BR",
|
|
"page_cycler_v2.intl_hi_ru",
|
|
"page_cycler_v2.intl_ja_zh",
|
|
"page_cycler_v2.intl_ko_th_vi",
|
|
"page_cycler_v2.netsim.top_10",
|
|
"page_cycler_v2.typical_25",
|
|
"spaceport",
|
|
"tab_switching.top_10",
|
|
]
|
|
telemetry_toolchain_perf_tests = [
|
|
"octane",
|
|
"kraken",
|
|
"speedometer",
|
|
"speedometer2",
|
|
"jetstream2",
|
|
]
|
|
graphics_perf_tests = [
|
|
"graphics_GLBench",
|
|
"graphics_GLMark2",
|
|
"graphics_SanAngeles",
|
|
"graphics_WebGLAquarium",
|
|
"graphics_WebGLPerformance",
|
|
]
|
|
# TODO: disable rendering.desktop by default as the benchmark is
|
|
# currently in a bad state
|
|
# page_cycler_v2.typical_25 is deprecated and the recommend replacement is
|
|
# loading.desktop@@typical (crbug.com/916340)
|
|
telemetry_crosbolt_perf_tests = [
|
|
"octane",
|
|
"kraken",
|
|
"speedometer2",
|
|
"jetstream",
|
|
"loading.desktop",
|
|
# 'rendering.desktop',
|
|
]
|
|
|
|
crosbolt_perf_tests = [
|
|
"graphics_WebGLAquarium",
|
|
"tast.video.PlaybackPerfVP91080P30FPS",
|
|
]
|
|
|
|
# 'cheets_AntutuTest',
|
|
# 'cheets_PerfBootServer',
|
|
# 'cheets_CandyCrushTest',
|
|
# 'cheets_LinpackTest',
|
|
# ]
|
|
|
|
dso_list = [
|
|
"all",
|
|
"chrome",
|
|
"kallsyms",
|
|
]
|
|
|
|
|
|
class ExperimentFactory(object):
|
|
"""Factory class for building an Experiment, given an ExperimentFile as input.
|
|
|
|
This factory is currently hardcoded to produce an experiment for running
|
|
ChromeOS benchmarks, but the idea is that in the future, other types
|
|
of experiments could be produced.
|
|
"""
|
|
|
|
def AppendBenchmarkSet(
|
|
self,
|
|
benchmarks,
|
|
benchmark_list,
|
|
test_args,
|
|
iterations,
|
|
rm_chroot_tmp,
|
|
perf_args,
|
|
suite,
|
|
show_all_results,
|
|
retries,
|
|
run_local,
|
|
cwp_dso,
|
|
weight,
|
|
):
|
|
"""Add all the tests in a set to the benchmarks list."""
|
|
for test_name in benchmark_list:
|
|
telemetry_benchmark = Benchmark(
|
|
test_name,
|
|
test_name,
|
|
test_args,
|
|
iterations,
|
|
rm_chroot_tmp,
|
|
perf_args,
|
|
suite,
|
|
show_all_results,
|
|
retries,
|
|
run_local,
|
|
cwp_dso,
|
|
weight,
|
|
)
|
|
benchmarks.append(telemetry_benchmark)
|
|
|
|
def GetExperiment(self, experiment_file, working_directory, log_dir):
|
|
"""Construct an experiment from an experiment file."""
|
|
global_settings = experiment_file.GetGlobalSettings()
|
|
experiment_name = global_settings.GetField("name")
|
|
board = global_settings.GetField("board")
|
|
chromeos_root = global_settings.GetField("chromeos_root")
|
|
log_level = global_settings.GetField("logging_level")
|
|
if log_level not in ("quiet", "average", "verbose"):
|
|
log_level = "verbose"
|
|
|
|
crosfleet = global_settings.GetField("crosfleet")
|
|
no_lock = bool(global_settings.GetField("no_lock"))
|
|
# Check whether crosfleet tool is installed correctly for crosfleet mode.
|
|
if crosfleet and not self.CheckCrosfleetTool(chromeos_root, log_level):
|
|
sys.exit(0)
|
|
|
|
remote = global_settings.GetField("remote")
|
|
# This is used to remove the ",' from the remote if user
|
|
# add them to the remote string.
|
|
new_remote = []
|
|
if remote:
|
|
for i in remote:
|
|
c = re.sub("[\"']", "", i)
|
|
new_remote.append(c)
|
|
remote = new_remote
|
|
rm_chroot_tmp = global_settings.GetField("rm_chroot_tmp")
|
|
perf_args = global_settings.GetField("perf_args")
|
|
download_debug = global_settings.GetField("download_debug")
|
|
# Do not download debug symbols when perf_args is not specified.
|
|
if not perf_args and download_debug:
|
|
download_debug = False
|
|
acquire_timeout = global_settings.GetField("acquire_timeout")
|
|
cache_dir = global_settings.GetField("cache_dir")
|
|
cache_only = global_settings.GetField("cache_only")
|
|
config.AddConfig("no_email", global_settings.GetField("no_email"))
|
|
share_cache = global_settings.GetField("share_cache")
|
|
results_dir = global_settings.GetField("results_dir")
|
|
compress_results = global_settings.GetField("compress_results")
|
|
# Warn user that option use_file_locks is deprecated.
|
|
use_file_locks = global_settings.GetField("use_file_locks")
|
|
if use_file_locks:
|
|
l = logger.GetLogger()
|
|
l.LogWarning(
|
|
"Option use_file_locks is deprecated, please remove it "
|
|
"from your experiment settings."
|
|
)
|
|
locks_dir = global_settings.GetField("locks_dir")
|
|
# If not specified, set the locks dir to the default locks dir in
|
|
# file_lock_machine.
|
|
if not locks_dir:
|
|
locks_dir = file_lock_machine.Machine.LOCKS_DIR
|
|
if not os.path.exists(locks_dir):
|
|
raise RuntimeError(
|
|
"Cannot access default lock directory. "
|
|
"Please run prodaccess or specify a local directory"
|
|
)
|
|
chrome_src = global_settings.GetField("chrome_src")
|
|
show_all_results = global_settings.GetField("show_all_results")
|
|
cwp_dso = global_settings.GetField("cwp_dso")
|
|
if cwp_dso and not cwp_dso in dso_list:
|
|
raise RuntimeError("The DSO specified is not supported")
|
|
ignore_min_max = global_settings.GetField("ignore_min_max")
|
|
dut_config = {
|
|
"enable_aslr": global_settings.GetField("enable_aslr"),
|
|
"intel_pstate": global_settings.GetField("intel_pstate"),
|
|
"cooldown_time": global_settings.GetField("cooldown_time"),
|
|
"cooldown_temp": global_settings.GetField("cooldown_temp"),
|
|
"governor": global_settings.GetField("governor"),
|
|
"cpu_usage": global_settings.GetField("cpu_usage"),
|
|
"cpu_freq_pct": global_settings.GetField("cpu_freq_pct"),
|
|
"turbostat": global_settings.GetField("turbostat"),
|
|
"top_interval": global_settings.GetField("top_interval"),
|
|
}
|
|
|
|
# Default cache hit conditions. The image checksum in the cache and the
|
|
# computed checksum of the image must match. Also a cache file must exist.
|
|
cache_conditions = [
|
|
CacheConditions.CACHE_FILE_EXISTS,
|
|
CacheConditions.CHECKSUMS_MATCH,
|
|
]
|
|
if global_settings.GetField("rerun_if_failed"):
|
|
cache_conditions.append(CacheConditions.RUN_SUCCEEDED)
|
|
if global_settings.GetField("rerun"):
|
|
cache_conditions.append(CacheConditions.FALSE)
|
|
if global_settings.GetField("same_machine"):
|
|
cache_conditions.append(CacheConditions.SAME_MACHINE_MATCH)
|
|
if global_settings.GetField("same_specs"):
|
|
cache_conditions.append(CacheConditions.MACHINES_MATCH)
|
|
|
|
# Construct benchmarks.
|
|
# Some fields are common with global settings. The values are
|
|
# inherited and/or merged with the global settings values.
|
|
benchmarks = []
|
|
all_benchmark_settings = experiment_file.GetSettings("benchmark")
|
|
|
|
# Check if there is duplicated benchmark name
|
|
benchmark_names = {}
|
|
# Check if in cwp_dso mode, all benchmarks should have same iterations
|
|
cwp_dso_iterations = 0
|
|
|
|
for benchmark_settings in all_benchmark_settings:
|
|
benchmark_name = benchmark_settings.name
|
|
test_name = benchmark_settings.GetField("test_name")
|
|
if not test_name:
|
|
test_name = benchmark_name
|
|
test_args = benchmark_settings.GetField("test_args")
|
|
|
|
# Rename benchmark name if 'story-filter' or 'story-tag-filter' specified
|
|
# in test_args. Make sure these two tags only appear once.
|
|
story_count = 0
|
|
for arg in test_args.split():
|
|
if "--story-filter=" in arg or "--story-tag-filter=" in arg:
|
|
story_count += 1
|
|
if story_count > 1:
|
|
raise RuntimeError(
|
|
"Only one story or story-tag filter allowed in "
|
|
"a single benchmark run"
|
|
)
|
|
# Rename benchmark name with an extension of 'story'-option
|
|
benchmark_name = "%s@@%s" % (
|
|
benchmark_name,
|
|
arg.split("=")[-1],
|
|
)
|
|
|
|
# Check for duplicated benchmark name after renaming
|
|
if not benchmark_name in benchmark_names:
|
|
benchmark_names[benchmark_name] = True
|
|
else:
|
|
raise SyntaxError(
|
|
"Duplicate benchmark name: '%s'." % benchmark_name
|
|
)
|
|
|
|
iterations = benchmark_settings.GetField("iterations")
|
|
if cwp_dso:
|
|
if cwp_dso_iterations not in (0, iterations):
|
|
raise RuntimeError(
|
|
"Iterations of each benchmark run are not the " "same"
|
|
)
|
|
cwp_dso_iterations = iterations
|
|
|
|
suite = benchmark_settings.GetField("suite")
|
|
retries = benchmark_settings.GetField("retries")
|
|
run_local = benchmark_settings.GetField("run_local")
|
|
weight = benchmark_settings.GetField("weight")
|
|
if weight:
|
|
if not cwp_dso:
|
|
raise RuntimeError(
|
|
"Weight can only be set when DSO specified"
|
|
)
|
|
if suite != "telemetry_Crosperf":
|
|
raise RuntimeError(
|
|
"CWP approximation weight only works with "
|
|
"telemetry_Crosperf suite"
|
|
)
|
|
if run_local:
|
|
raise RuntimeError(
|
|
"run_local must be set to False to use CWP "
|
|
"approximation"
|
|
)
|
|
if weight < 0:
|
|
raise RuntimeError("Weight should be a float >=0")
|
|
elif cwp_dso:
|
|
raise RuntimeError(
|
|
"With DSO specified, each benchmark should have a " "weight"
|
|
)
|
|
|
|
if suite == "telemetry_Crosperf":
|
|
if test_name == "all_perfv2":
|
|
self.AppendBenchmarkSet(
|
|
benchmarks,
|
|
telemetry_perfv2_tests,
|
|
test_args,
|
|
iterations,
|
|
rm_chroot_tmp,
|
|
perf_args,
|
|
suite,
|
|
show_all_results,
|
|
retries,
|
|
run_local,
|
|
cwp_dso,
|
|
weight,
|
|
)
|
|
elif test_name == "all_pagecyclers":
|
|
self.AppendBenchmarkSet(
|
|
benchmarks,
|
|
telemetry_pagecycler_tests,
|
|
test_args,
|
|
iterations,
|
|
rm_chroot_tmp,
|
|
perf_args,
|
|
suite,
|
|
show_all_results,
|
|
retries,
|
|
run_local,
|
|
cwp_dso,
|
|
weight,
|
|
)
|
|
elif test_name == "all_crosbolt_perf":
|
|
self.AppendBenchmarkSet(
|
|
benchmarks,
|
|
telemetry_crosbolt_perf_tests,
|
|
test_args,
|
|
iterations,
|
|
rm_chroot_tmp,
|
|
perf_args,
|
|
"telemetry_Crosperf",
|
|
show_all_results,
|
|
retries,
|
|
run_local,
|
|
cwp_dso,
|
|
weight,
|
|
)
|
|
self.AppendBenchmarkSet(
|
|
benchmarks,
|
|
crosbolt_perf_tests,
|
|
"",
|
|
iterations,
|
|
rm_chroot_tmp,
|
|
perf_args,
|
|
"",
|
|
show_all_results,
|
|
retries,
|
|
run_local=False,
|
|
cwp_dso=cwp_dso,
|
|
weight=weight,
|
|
)
|
|
elif test_name == "all_toolchain_perf":
|
|
self.AppendBenchmarkSet(
|
|
benchmarks,
|
|
telemetry_toolchain_perf_tests,
|
|
test_args,
|
|
iterations,
|
|
rm_chroot_tmp,
|
|
perf_args,
|
|
suite,
|
|
show_all_results,
|
|
retries,
|
|
run_local,
|
|
cwp_dso,
|
|
weight,
|
|
)
|
|
# Add non-telemetry toolchain-perf benchmarks:
|
|
|
|
# Tast test platform.ReportDiskUsage for image size.
|
|
benchmarks.append(
|
|
Benchmark(
|
|
"platform.ReportDiskUsage",
|
|
"platform.ReportDiskUsage",
|
|
"",
|
|
1, # This is not a performance benchmark, only run once.
|
|
rm_chroot_tmp,
|
|
"",
|
|
"tast", # Specify the suite to be 'tast'
|
|
show_all_results,
|
|
retries,
|
|
)
|
|
)
|
|
|
|
# TODO: crbug.com/1057755 Do not enable graphics_WebGLAquarium until
|
|
# it gets fixed.
|
|
#
|
|
# benchmarks.append(
|
|
# Benchmark(
|
|
# 'graphics_WebGLAquarium',
|
|
# 'graphics_WebGLAquarium',
|
|
# '',
|
|
# iterations,
|
|
# rm_chroot_tmp,
|
|
# perf_args,
|
|
# 'crosperf_Wrapper', # Use client wrapper in Autotest
|
|
# show_all_results,
|
|
# retries,
|
|
# run_local=False,
|
|
# cwp_dso=cwp_dso,
|
|
# weight=weight))
|
|
elif test_name == "all_toolchain_perf_old":
|
|
self.AppendBenchmarkSet(
|
|
benchmarks,
|
|
telemetry_toolchain_old_perf_tests,
|
|
test_args,
|
|
iterations,
|
|
rm_chroot_tmp,
|
|
perf_args,
|
|
suite,
|
|
show_all_results,
|
|
retries,
|
|
run_local,
|
|
cwp_dso,
|
|
weight,
|
|
)
|
|
else:
|
|
benchmark = Benchmark(
|
|
benchmark_name,
|
|
test_name,
|
|
test_args,
|
|
iterations,
|
|
rm_chroot_tmp,
|
|
perf_args,
|
|
suite,
|
|
show_all_results,
|
|
retries,
|
|
run_local,
|
|
cwp_dso,
|
|
weight,
|
|
)
|
|
benchmarks.append(benchmark)
|
|
else:
|
|
if test_name == "all_graphics_perf":
|
|
self.AppendBenchmarkSet(
|
|
benchmarks,
|
|
graphics_perf_tests,
|
|
"",
|
|
iterations,
|
|
rm_chroot_tmp,
|
|
perf_args,
|
|
"",
|
|
show_all_results,
|
|
retries,
|
|
run_local=False,
|
|
cwp_dso=cwp_dso,
|
|
weight=weight,
|
|
)
|
|
else:
|
|
# Add the single benchmark.
|
|
benchmark = Benchmark(
|
|
benchmark_name,
|
|
test_name,
|
|
test_args,
|
|
iterations,
|
|
rm_chroot_tmp,
|
|
perf_args,
|
|
suite,
|
|
show_all_results,
|
|
retries,
|
|
run_local=False,
|
|
cwp_dso=cwp_dso,
|
|
weight=weight,
|
|
)
|
|
benchmarks.append(benchmark)
|
|
|
|
if not benchmarks:
|
|
raise RuntimeError("No benchmarks specified")
|
|
|
|
# Construct labels.
|
|
# Some fields are common with global settings. The values are
|
|
# inherited and/or merged with the global settings values.
|
|
labels = []
|
|
all_label_settings = experiment_file.GetSettings("label")
|
|
all_remote = list(remote)
|
|
for label_settings in all_label_settings:
|
|
label_name = label_settings.name
|
|
image = label_settings.GetField("chromeos_image")
|
|
build = label_settings.GetField("build")
|
|
autotest_path = label_settings.GetField("autotest_path")
|
|
debug_path = label_settings.GetField("debug_path")
|
|
chromeos_root = label_settings.GetField("chromeos_root")
|
|
my_remote = label_settings.GetField("remote")
|
|
compiler = label_settings.GetField("compiler")
|
|
new_remote = []
|
|
if my_remote:
|
|
for i in my_remote:
|
|
c = re.sub("[\"']", "", i)
|
|
new_remote.append(c)
|
|
my_remote = new_remote
|
|
|
|
if image:
|
|
if crosfleet:
|
|
raise RuntimeError(
|
|
"In crosfleet mode, local image should not be used."
|
|
)
|
|
if build:
|
|
raise RuntimeError(
|
|
"Image path and build are provided at the same "
|
|
"time, please use only one of them."
|
|
)
|
|
else:
|
|
if not build:
|
|
raise RuntimeError("Can not have empty 'build' field!")
|
|
image, autotest_path, debug_path = label_settings.GetXbuddyPath(
|
|
build,
|
|
autotest_path,
|
|
debug_path,
|
|
board,
|
|
chromeos_root,
|
|
log_level,
|
|
download_debug,
|
|
)
|
|
|
|
cache_dir = label_settings.GetField("cache_dir")
|
|
chrome_src = label_settings.GetField("chrome_src")
|
|
|
|
# TODO(yunlian): We should consolidate code in machine_manager.py
|
|
# to derermine whether we are running from within google or not
|
|
if (
|
|
"corp.google.com" in socket.gethostname()
|
|
and not my_remote
|
|
and not crosfleet
|
|
):
|
|
my_remote = self.GetDefaultRemotes(board)
|
|
if global_settings.GetField("same_machine") and len(my_remote) > 1:
|
|
raise RuntimeError(
|
|
"Only one remote is allowed when same_machine "
|
|
"is turned on"
|
|
)
|
|
all_remote += my_remote
|
|
image_args = label_settings.GetField("image_args")
|
|
if test_flag.GetTestMode():
|
|
# pylint: disable=too-many-function-args
|
|
label = MockLabel(
|
|
label_name,
|
|
build,
|
|
image,
|
|
autotest_path,
|
|
debug_path,
|
|
chromeos_root,
|
|
board,
|
|
my_remote,
|
|
image_args,
|
|
cache_dir,
|
|
cache_only,
|
|
log_level,
|
|
compiler,
|
|
crosfleet,
|
|
chrome_src,
|
|
)
|
|
else:
|
|
label = Label(
|
|
label_name,
|
|
build,
|
|
image,
|
|
autotest_path,
|
|
debug_path,
|
|
chromeos_root,
|
|
board,
|
|
my_remote,
|
|
image_args,
|
|
cache_dir,
|
|
cache_only,
|
|
log_level,
|
|
compiler,
|
|
crosfleet,
|
|
chrome_src,
|
|
)
|
|
labels.append(label)
|
|
|
|
if not labels:
|
|
raise RuntimeError("No labels specified")
|
|
|
|
email = global_settings.GetField("email")
|
|
all_remote += list(set(my_remote))
|
|
all_remote = list(set(all_remote))
|
|
if crosfleet:
|
|
for remote in all_remote:
|
|
self.CheckRemotesInCrosfleet(remote)
|
|
experiment = Experiment(
|
|
experiment_name,
|
|
all_remote,
|
|
working_directory,
|
|
chromeos_root,
|
|
cache_conditions,
|
|
labels,
|
|
benchmarks,
|
|
experiment_file.Canonicalize(),
|
|
email,
|
|
acquire_timeout,
|
|
log_dir,
|
|
log_level,
|
|
share_cache,
|
|
results_dir,
|
|
compress_results,
|
|
locks_dir,
|
|
cwp_dso,
|
|
ignore_min_max,
|
|
crosfleet,
|
|
dut_config,
|
|
no_lock=no_lock,
|
|
)
|
|
|
|
return experiment
|
|
|
|
def GetDefaultRemotes(self, board):
|
|
default_remotes_file = os.path.join(
|
|
os.path.dirname(__file__), "default_remotes"
|
|
)
|
|
try:
|
|
with open(default_remotes_file) as f:
|
|
for line in f:
|
|
key, v = line.split(":")
|
|
if key.strip() == board:
|
|
remotes = v.strip().split()
|
|
if remotes:
|
|
return remotes
|
|
else:
|
|
raise RuntimeError(
|
|
"There is no remote for {0}".format(board)
|
|
)
|
|
except IOError:
|
|
# TODO: rethrow instead of throwing different exception.
|
|
raise RuntimeError(
|
|
"IOError while reading file {0}".format(default_remotes_file)
|
|
)
|
|
else:
|
|
raise RuntimeError("There is no remote for {0}".format(board))
|
|
|
|
def CheckRemotesInCrosfleet(self, remote):
|
|
# TODO: (AI:zhizhouy) need to check whether a remote is a local or lab
|
|
# machine. If not lab machine, raise an error.
|
|
pass
|
|
|
|
def CheckCrosfleetTool(self, chromeos_root, log_level):
|
|
CROSFLEET_PATH = "crosfleet"
|
|
if os.path.exists(CROSFLEET_PATH):
|
|
return True
|
|
l = logger.GetLogger()
|
|
l.LogOutput("Crosfleet tool not installed, trying to install it.")
|
|
ce = command_executer.GetCommandExecuter(l, log_level=log_level)
|
|
setup_lab_tools = os.path.join(
|
|
chromeos_root, "chromeos-admin", "lab-tools", "setup_lab_tools"
|
|
)
|
|
cmd = "%s" % setup_lab_tools
|
|
status = ce.RunCommand(cmd)
|
|
if status != 0:
|
|
raise RuntimeError(
|
|
"Crosfleet tool not installed correctly, please try to "
|
|
"manually install it from %s" % setup_lab_tools
|
|
)
|
|
l.LogOutput(
|
|
"Crosfleet is installed at %s, please login before first use. "
|
|
'Login by running "crosfleet login" and follow instructions.'
|
|
% CROSFLEET_PATH
|
|
)
|
|
return False
|