diff --git a/CHANGES b/CHANGES
index ec6ad4b61..a96335e6e 100644
--- a/CHANGES
+++ b/CHANGES
@@ -22,8 +22,16 @@ This release contains contributions from (alphabetically by first name):
- The *machineid* module, which generates UUIDs for systemd and dbus
and can generate entropy files (filled from `/dev/urandom` in the host
system) now supports more than one entropy file; generate them as needed
- (or copy a fixed value to all, depending on *entropy-copy*). Deprecate
- *entropy* (which generates a specific output file) as too inflexible.
+ (or copy a fixed value to all, depending on *entropy-copy*). Deprecate
+ *entropy* (which generates a specific output file) as too inflexible.
+ - Progress reporting from the *unpackfs* module has been revamped:
+ it reports more often now, so that it is more obvious that files
+ are being transferred even when the percentage progress does not
+ change.
+ - The *unpackfs* module now supports a *weight* setting for each
+ of the unpack entries. For a single entry this does not matter,
+ but if there are multiple entries it allows tweaking the relative
+ progress between each entry.
# 3.2.30 (2020-09-03) #
diff --git a/src/modules/unpackfs/main.py b/src/modules/unpackfs/main.py
index d89607465..a573cf6e7 100644
--- a/src/modules/unpackfs/main.py
+++ b/src/modules/unpackfs/main.py
@@ -48,8 +48,8 @@ class UnpackEntry:
:param sourcefs:
:param destination:
"""
- __slots__ = ['source', 'sourcefs', 'destination', 'copied', 'total', 'exclude', 'excludeFile',
- 'mountPoint']
+ __slots__ = ('source', 'sourcefs', 'destination', 'copied', 'total', 'exclude', 'excludeFile',
+ 'mountPoint', 'weight')
def __init__(self, source, sourcefs, destination):
"""
@@ -71,6 +71,7 @@ class UnpackEntry:
self.copied = 0
self.total = 0
self.mountPoint = None
+ self.weight = 1
def is_file(self):
return self.sourcefs == "file"
@@ -162,6 +163,8 @@ def file_copy(source, entry, progress_cb):
:param progress_cb: A callback function for progress reporting.
Takes a number and a total-number.
"""
+ import time
+
dest = entry.destination
# Environment used for executing rsync properly
@@ -190,12 +193,14 @@ def file_copy(source, entry, progress_cb):
args, env=at_env,
stdout=subprocess.PIPE, close_fds=ON_POSIX
)
- # last_num_files_copied trails num_files_copied, and whenever at least 100 more
- # files have been copied, progress is reported and last_num_files_copied is updated.
+ # last_num_files_copied trails num_files_copied, and whenever at least 107 more
+ # files (file_count_chunk) have been copied, progress is reported and
+ # last_num_files_copied is updated. The chunk size isn't "tidy"
+ # so that all the digits of the progress-reported number change.
+ #
last_num_files_copied = 0
- file_count_chunk = entry.total / 100
- if file_count_chunk < 100:
- file_count_chunk = 100
+ last_timestamp_reported = time.time()
+ file_count_chunk = 107
for line in iter(process.stdout.readline, b''):
# rsync outputs progress in parentheses. Each line will have an
@@ -220,9 +225,10 @@ def file_copy(source, entry, progress_cb):
# adjusting the offset so that progressbar can be continuesly drawn
num_files_copied = num_files_total_local - num_files_remaining
- # Update about once every 1% of this entry
- if num_files_copied - last_num_files_copied >= file_count_chunk:
+ now = time.time()
+ if (num_files_copied - last_num_files_copied >= file_count_chunk) or (now - last_timestamp_reported > 0.5):
last_num_files_copied = num_files_copied
+ last_timestamp_reported = now
progress_cb(num_files_copied, num_files_total_local)
process.wait()
@@ -260,6 +266,7 @@ class UnpackOperation:
def __init__(self, entries):
self.entries = entries
self.entry_for_source = dict((x.source, x) for x in self.entries)
+ self.total_weight = sum([e.weight for e in entries])
def report_progress(self):
"""
@@ -267,30 +274,29 @@ class UnpackOperation:
"""
progress = float(0)
- done = 0 # Done and total apply to the entry now-unpacking
- total = 0
- complete = 0 # This many are already finished
+ current_total = 0
+ current_done = 0 # Files count in the current entry
+ complete_count = 0
+ complete_weight = 0 # This much weight already finished
for entry in self.entries:
if entry.total == 0:
# Total 0 hasn't counted yet
continue
if entry.total == entry.copied:
- complete += 1
+ complete_weight += entry.weight
+ complete_count += 1
else:
# There is at most *one* entry in-progress
- total = entry.total
- done = entry.copied
+ current_total = entry.total
+ current_done = entry.copied
+ complete_weight += entry.weight * ( 1.0 * current_done ) / current_total
break
- if total > 0:
- # Pretend that each entry represents an equal amount of work;
- # the complete ones count as 100% of their own fraction
- # (and have *not* been counted in total or done), while
- # total/done represents the fraction of the current fraction.
- progress = ( ( 1.0 * complete ) / len(self.entries) ) + ( ( 1.0 / len(self.entries) ) * ( 1.0 * done / total ) )
+ if current_total > 0:
+ progress = ( 1.0 * complete_weight ) / self.total_weight
global status
- status = _("Unpacking image {}/{}, file {}/{}").format((complete+1),len(self.entries),done, total)
+ status = _("Unpacking image {}/{}, file {}/{}").format((complete_count+1), len(self.entries), current_done, current_total)
job.setprogress(progress)
def run(self):
@@ -395,6 +401,24 @@ def repair_root_permissions(root_mount_point):
# But ignore it
+def extract_weight(entry):
+ """
+ Given @p entry, a dict representing a single entry in
+ the *unpack* list, returns its weight (1, or whatever is
+ set if it is sensible).
+ """
+ w = entry.get("weight", None)
+ if w:
+ try:
+ wi = int(w)
+ return wi if wi > 0 else 1
+ except ValueError:
+ utils.warning("*weight* setting {!r} is not valid.".format(w))
+ except TypeError:
+ utils.warning("*weight* setting {!r} must be number.".format(w))
+ return 1
+
+
def run():
"""
Unsquash filesystem.
@@ -461,6 +485,7 @@ def run():
unpack[-1].exclude = entry["exclude"]
if entry.get("excludeFile", None):
unpack[-1].excludeFile = entry["excludeFile"]
+ unpack[-1].weight = extract_weight(entry)
is_first = False
diff --git a/src/modules/unpackfs/unpackfs.conf b/src/modules/unpackfs/unpackfs.conf
index 2c4a25a80..d12110b60 100644
--- a/src/modules/unpackfs/unpackfs.conf
+++ b/src/modules/unpackfs/unpackfs.conf
@@ -32,6 +32,12 @@
# - *excludeFile* is a single file that is passed to rsync as an
# --exclude-file argument. This should be a full pathname
# inside the **host** filesystem.
+# - *weight* is useful when the entries take wildly different
+# times to unpack (e.g. with a squashfs, and one single file)
+# and the total weight of this module should be distributed
+# differently between the entries. (This is only relevant when
+# there is more than one entry; by default all the entries
+# have the same weight, 1)
#
# EXAMPLES
#
@@ -85,8 +91,10 @@ unpack:
- source: ../CHANGES
sourcefs: file
destination: "/tmp/changes.txt"
+ weight: 1 # Single file
- source: src/qml/calamares/slideshow
sourcefs: file
destination: "/tmp/slideshow/"
exclude: [ "*.qmlc", "qmldir" ]
+ weight: 5 # Lots of files
# excludeFile: /etc/calamares/modules/unpackfs/exclude-list.txt