BENCHMARK_TYPES = ["dummy", "text"]
DEFAULT_BUCKET = "lava"
+invalid_commits = {
+ "ec9a9794af488a9accce7708a8b0d8188b498789", # Does not build
+ "8c99128c640cbce71fb8a6caa15e4c672252b662", # Block on configure
+ "f3847c753f1b4f12353c38d97b0577d9993d19fb", # Does not build
+ "e0111295f17ddfcc33ec771a8deac505473a06ad", # Does not build
+ "d0d4e0ed487ea23aaf0d023513c0a4d86901b79b", # Does not build
+ "c24f7ab4dd9edeb5e50b0070fd9d9e8691057dde", # Does not build
+ "ce67f5614a4db3b2de4d887eca52135b439b4937", # Does not build
+ "80aff5efc66679fd934cef433c0e698694748385", # Does not build
+ "f4f11e84942d36fcc8a597d226928bce2ccac4b3", # Does not build
+ "ae466a6e1b856d96cf5112a371b4df2b732503ec", # Does not build
+ "ade5c95e2a4f90f839f222fc1a66175b3b199922", # Configuration fails
+ "30341532906d62808e9d66fb115f5edb4e6f5706", # Configuration fails
+ "006c5ffb42f32e802136e3c27a63accb59b4d6c4", # Does not build
+ "88488ff5bdcd7679ff1f04fe6cff0d24b4f8fc0c", # Does not build
+ # Other errors
+ "7c7301d5827bd10ec7c34da7ffc5fe74e5047d38",
+ "a0df3abf88616cb0799f87f4eb57c54268e63448",
+ "b7045dd71bc0524ad6b5db96df365e98e237d395",
+ "cf7b259eaa602abcef308d2b5dd8e6c9ee995d8b",
+ "90a55a4ef47cac7b568f5f0a8a78bd760f82d23c",
+ "baa5e3aa82a82c9d0fa59e3c586c0168bb5dc267",
+ "af9f8da7ba4a9b16fc36d637b8c3a0c7a8774da2",
+ "fe748379adbd385efdfc7acae9c2340fb8b7d717",
+ "baa5e3aa82a82c9d0fa59e3c586c0168bb5dc267",
+ "af9f8da7ba4a9b16fc36d637b8c3a0c7a8774da2",
+ "fe748379adbd385efdfc7acae9c2340fb8b7d717",
+ "929627965e33e06dc77254d81e8ec1d66cc06590",
+ "48a0e52c4632a60cd43423f2f34f10de350bf868",
+ "b7fa35fce415b33207a9eba111069ed31ef122a0",
+ "828c8a25785e0cedaeb6987256a4dfc3c43b982f",
+ "213489680861e4d796173513effac7023312ec2d",
+ "430a5ccbbd15782501ca56bb148f3850126277ad",
+ "629d19044c43b195498d0a4e002906c54b6186d5",
+ "c423217ed1640b4152739f7e5613775d46c25050",
+ # Elfutils
+ "776a2a252c9875caa1e8b4f41cb8cc12c79611c3",
+ "435aa29aff0527d36aafa1b657ae70b9db5f9ea5",
+ "95651695473495501fc6b2c4a1cf6a78cfb3cd6a",
+ "e0748fb2ba8994c136bcc0b67d3044f09841cf8e",
+ "9e632b22e1310fe773edc32ab08a60602f4b2861",
+ "271fb6907a6f4705a1c799d925394243eae51d68",
+ "328342cd737582216dc7b8b7d558b2a1bf8ea5e8",
+ "ae5c1a4481be68fae027910b141354c1d86daa64",
+ "e6938018975e45d35dab5fef795fe7344eef7d62",
+ "e015bae2ef343b30c890eebb9182a8be13d12ed0",
+ "5e8a0751ae0c418a615025d1da10bc84f91b3d97",
+ "887d26fa0fd0ae0c5c15e4b885473c4cdc0bf078",
+ "e97fe75eac59fc39a6e4f3c4f9f3301835a0315e",
+ "8b130e7f1d6a41fb5c64a014c15246ba74b79470",
+ "f4f8f79893b18199b38edc3330093a9403c4c737",
+}
+
+def json_type(string):
+ """
+ Argpase type for json args.
+ We expect a base dictionary.
+ """
+ passed_json = json.loads(string)
+ if not isinstance(passed_json, dict):
+ msg = "%r is not a dict" % string
+ raise argparse.ArgumentTypeError(msg)
+ return passed_json
def graph_get_color(branch):
"""
results = {}
benchmark_valid = True
for b_type in BENCHMARK_TYPES:
- prefix = "/results/benchmarks/babeltrace/{}/".format(b_type)
+ prefix = "/results/benchmarks/babeltrace/{}".format(b_type)
result_file = get_file(client, prefix, commit, workdir)
if not result_file:
"""
)
plt.plot(outlier_x_data, outlier_y_data, "+", label="outlier", color="black")
- ymin = 0
ymax = 1
if y_data:
ymin = 0.8 * min([item for sublist in y_data for item in sublist])
label="Latest {}".format(l_branch),
color=graph_get_color(l_branch),
)
- if l_result <= ymin:
- ymin = 0.8 * l_result
if l_result >= ymax:
ymax = 1.2 * l_result
-
- plt.ylim(ymin=ymin, ymax=ymax)
+ ax = plt.gca()
+ plt.ylim(ymin=0, ymax=ymax)
plt.xticks(x_data, labels, rotation=90, family="monospace")
plt.title(graph_get_title(branch, benchmark_type), fontweight="bold")
plt.ylabel("User + system time (s)")
plt.xlabel("Latest commits")
plt.legend()
+ plt.grid(True)
+
+ # Put tick on the right side
+ ax.tick_params(labeltop=False, labelright=True)
+
+ plt.tight_layout()
+ return
+
+
+def plot_delta_between_point(
+ branch, benchmark_type, x_data, y_data, labels, latest_values
+):
+ """
+ Plot the graph of delta between each sequential commit.
+ """
+ local_abs_max = 100
+
+ # Transform y_data to a list of for which the reference is the first
+ # element.
+ local_y_data = []
+ for pos, y in enumerate(y_data):
+ if pos == 0:
+ local_y_data.append(0.0)
+ continue
+ local_y_data.append(y - y_data[pos - 1])
+
+ plt.plot(x_data, local_y_data, "o", label=branch, color=graph_get_color(branch))
+
+ # Get max absolute value to align the y axis with zero in the middle.
+ if local_y_data:
+ local_abs_max = abs(max(local_y_data, key=abs)) * 1.3
+
+ plt.ylim(ymin=local_abs_max * -1, ymax=local_abs_max)
+
+ ax = plt.gca()
+ plt.xticks(x_data, labels, rotation=90, family="monospace")
+ plt.title(
+ graph_get_title(branch, benchmark_type) + " Delta to previous commit",
+ fontweight="bold",
+ )
+ plt.ylabel("Seconds")
+ plt.xlabel("Latest commits")
+ plt.legend()
+ plt.grid(True)
+
+ # Put tick on the right side
+ ax.tick_params(labeltop=False, labelright=True)
plt.tight_layout()
return
plt.ylabel("Ratio")
plt.xlabel("Latest commits")
plt.legend()
+ plt.grid(True)
+
+ # Put tick on the right side
+ ax.tick_params(labeltop=False, labelright=True)
plt.tight_layout()
return
plot_raw_value(branch, b_type, x_data, y_data, labels, latest_values)
pdf_pages.savefig(fig)
- fig = plt.figure(figsize=(width, 8.27), dpi=100)
# Use the mean of each sanitize dataset here, we do not care for
# variance for ratio. At least not yet.
y_data = [mean(sanitize_dataset(c[1][b_type])[0]) for c in results]
+ fig = plt.figure(figsize=(width, 8.27), dpi=100)
plot_ratio(branch, b_type, x_data, y_data, labels, latest_values)
pdf_pages.savefig(fig)
+ fig = plt.figure(figsize=(width, 8.27), dpi=100)
+ plot_delta_between_point(
+ branch, b_type, x_data, y_data, labels, latest_values
+ )
+ pdf_pages.savefig(fig)
+
pdf_pages.close()
-def launch_jobs(branches, git_path, wait_for_completion, debug):
+def launch_jobs(branches, git_path, wait_for_completion, debug, force):
"""
Lauch jobs for all missing results.
"""
client = get_client()
+ commits_to_test = set()
for branch, cutoff in branches.items():
- commits = get_git_log(branch, cutoff, git_path)
-
+ commits = [x for x in get_git_log(branch, cutoff, git_path) if x not in invalid_commits]
with tempfile.TemporaryDirectory() as workdir:
for commit in commits:
b_results = get_benchmark_results(client, commit, workdir)[0]
- if b_results:
+ if b_results and not force:
continue
- lava_submit.submit(
- commit, wait_for_completion=wait_for_completion, debug=debug
- )
+ commits_to_test.add(commit)
+ for index, commit in enumerate(commits_to_test):
+ print("Job {}/{}".format(index+1, len(commits_to_test)))
+ lava_submit.submit(
+ commit, wait_for_completion=wait_for_completion, debug=debug
+ )
def main():
parser.add_argument(
"--generate-jobs", action="store_true", help="Generate and send jobs"
)
+ parser.add_argument(
+ "--force-jobs", action="store_true", help="Force the queueing of jobs to lava"
+ )
parser.add_argument(
"--do-not-wait-on-completion",
action="store_true",
parser.add_argument(
"--repo-path", help="The location of the git repo to use.", required=True
)
+ parser.add_argument(
+ "--overwrite-branches-cutoff",
+ help="A dictionary of the form {"
+ "'branch_name': 'commit_hash_cutoff',...}. Allow custom graphing and"
+ "jobs generation.",
+ required=False, type=json_type
+ )
args = parser.parse_args()
+ if args.overwrite_branches_cutoff:
+ bt_branches = args.overwrite_branches_cutoff
+
if not os.path.exists(args.repo_path):
print("Repository location does not exists.")
return 1
if args.generate_jobs:
print("Launching jobs for:")
+
for branch, cutoff in bt_branches.items():
print("\t Branch {} with cutoff {}".format(branch, cutoff))
+
launch_jobs(
- bt_branches, args.repo_path, not args.do_not_wait_on_completion, args.debug
+ bt_branches,
+ args.repo_path,
+ not args.do_not_wait_on_completion,
+ args.debug,
+ args.force_jobs,
)
if args.generate_report:
projects / lttng-ci.git / blobdiff