From 6f6f5e70f16bef21523c3e2f19e7557bfcaa2546 Mon Sep 17 00:00:00 2001
From: Michal Domonkos <mdomonko@redhat.com>
Date: Tue, 21 Apr 2020 11:38:25 +0200
Subject: [PATCH] build: prioritize large packages

Binary packages come in different sizes and so their build time can vary
greatly.  Dynamic scheduling, which we currently use for parallel
building, is a good strategy to combat such differences and load-balance
the available CPU cores.

That said, knowing that the build time of a package is proportional to
its size, we can reduce the overall time even further by cleverly
ordering the task queue.

As an example, consider a set of 5 packages, 4 of which take 1 unit of
time to build and one takes 4 units.  If we were to build these on a
dual-core system, one possible unit distribution would look like this:

            TIME --->
CPU 1       * * * * * *         # package 1, 3 and 5
CPU 2       * *                 # package 2 and 4

Now, compare that to a different distribution where the largest package
5 gets built early on:

            TIME --->
CPU 1       * * * *             # package 5
CPU 2       * * * *             # package 1, 2, 3 and 4

It's obvious that processing the largest packages first gives better
results when dealing with such a mix of small and large packages
(typically a regular package and its debuginfo counterpart,
respectively).

Now, with dynamic scheduling in OpenMP, we cannot directly control the
task queue; we can only generate the tasks and let the runtime system do
its work.  What we can do, however, is to provide a hint to the runtime
system for the desired ordering, using the "priority" clause.

So, in this commit, we use the clause to assign a priority value to each
build task based on the respective package size (the bigger the size,
the higher the priority), to help achieve an optimal execution order.

Indeed, in my testing, the priorities were followed to the letter (but
remember, that's not guaranteed by the specification).  Interestingly,
even without the use of priorities, simply generating the tasks in the
desired order resulted in the same execution order for me, but that's,
again, just an implementation detail.

Also note that OpenMP is allowed to stop the thread generating the tasks
at any time, and make it execute some of the tasks instead.  If the
chosen task happens to be a long-duration one, we might hit a starvation
scenario where the other threads have exhausted the task queue and
there's nobody to generate new tasks.  To counter that, this commit also
adds the "untied" clause which allows other threads to pick up where the
generating thread left off, and continue generating new tasks.

Resolves #1045.
---
 build/pack.c | 38 +++++++++++++++++++++++++++++++++++---
 1 file changed, 35 insertions(+), 3 deletions(-)

diff --git a/build/pack.c b/build/pack.c
index a44a3fe9c8..bc40683c4f 100644
--- a/build/pack.c
+++ b/build/pack.c
@@ -6,6 +6,7 @@
 #include "system.h"
 
 #include <errno.h>
+#include <stdlib.h>
 #include <sys/wait.h>
 
 #include <rpm/rpmlib.h>			/* RPMSIGTAG*, rpmReadPackageFile */
@@ -726,16 +727,45 @@ static rpmRC packageBinary(rpmSpec spec, Package pkg, const char *cookie, int ch
     return rc;
 }
 
+static int compareBinaries(const void *p1, const void *p2) {
+    Package pkg1 = *(Package *)p1;
+    Package pkg2 = *(Package *)p2;
+    uint64_t size1 = headerGetNumber(pkg1->header, RPMTAG_LONGSIZE);
+    uint64_t size2 = headerGetNumber(pkg2->header, RPMTAG_LONGSIZE);
+    if (size1 > size2)
+        return -1;
+    if (size1 < size2)
+        return 1;
+    return 0;
+}
+
+/*
+ * Run binary creation in parallel, with task priority based on package size
+ * (largest first) to help achieve an optimal load distribution.
+ */
 rpmRC packageBinaries(rpmSpec spec, const char *cookie, int cheating)
 {
     rpmRC rc = RPMRC_OK;
     Package pkg;
+    Package *tasks;
+    int npkgs = 0;
+
+    for (pkg = spec->packages; pkg != NULL; pkg = pkg->next)
+        npkgs++;
+    tasks = xcalloc(npkgs, sizeof(Package));
+
+    pkg = spec->packages;
+    for (int i = 0; i < npkgs; i++) {
+        tasks[i] = pkg;
+        pkg = pkg->next;
+    }
+    qsort(tasks, npkgs, sizeof(Package), compareBinaries);
 
-    /* Run binary creation in parallel */
     #pragma omp parallel
     #pragma omp single
-    for (pkg = spec->packages; pkg != NULL; pkg = pkg->next) {
-	#pragma omp task
+    for (int i = 0; i < npkgs; i++) {
+	pkg = tasks[i];
+	#pragma omp task untied priority(i)
 	{
 	pkg->rc = packageBinary(spec, pkg, cookie, cheating, &pkg->filename);
 	rpmlog(RPMLOG_DEBUG,
@@ -754,6 +784,8 @@ rpmRC packageBinaries(rpmSpec spec, const char *cookie, int cheating)
     if (rc == RPMRC_OK)
 	checkPackageSet(spec->packages);
 
+    free(tasks);
+
     return rc;
 }