From 4e6492de28032f25fc53e4dc3862dc79c3cc42b6 Mon Sep 17 00:00:00 2001
From: Arnaud Ferraris <arnaud.ferraris@collabora.com>
Date: Wed, 23 Jan 2019 16:36:31 +0100
Subject: [PATCH] Add a raw filesystem copy module
In some cases, we might want to copy a filesystem as if we were using a
simple 'dd' command, in order to create an exact copy, down to the block
level.
This can be useful in particular when working with dm-verity for
checking the rootfs integrity: that way, we can make a direct copy of
the rootfs and its verity partition and keep the system usable.
This patch adds a new 'rawfs' module to calamares, making possible to
block-copy a filesystem to a block device.
Signed-off-by: Arnaud Ferraris <arnaud.ferraris@collabora.com>
---
src/modules/rawfs/main.py | 184 ++++++++++++++++++++++++++++++++++
src/modules/rawfs/module.desc | 7 ++
src/modules/rawfs/rawfs.conf | 24 +++++
3 files changed, 215 insertions(+)
create mode 100644 src/modules/rawfs/main.py
create mode 100644 src/modules/rawfs/module.desc
create mode 100644 src/modules/rawfs/rawfs.conf
diff --git a/src/modules/rawfs/main.py b/src/modules/rawfs/main.py
new file mode 100644
index 000000000..6ff258918
--- /dev/null
+++ b/src/modules/rawfs/main.py
@@ -0,0 +1,184 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# === This file is part of Calamares - <https://github.com/calamares> ===
+#
+# Copyright 2014, Teo Mrnjavac <teo@kde.org>
+# Copyright 2017, Alf Gaida <agaida@siduction.org>
+# Copyright 2017, Adriaan de Groot <groot@kde.org>
+# Copyright 2019, Collabora Ltd
+#
+# Calamares is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Calamares is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Calamares. If not, see <http://www.gnu.org/licenses/>.
+
+import libcalamares
+import os
+import stat
+import subprocess
+from time import gmtime, strftime, sleep
+from math import gcd
+
+import gettext
+_ = gettext.translation("calamares-python",
+ localedir=libcalamares.utils.gettext_path(),
+ languages=libcalamares.utils.gettext_languages(),
+ fallback=True).gettext
+
+def pretty_name():
+ return _("Installing data.")
+
+def lcm(a, b):
+ """
+ Computes the Least Common Multiple of 2 numbers
+ """
+ return a * b / gcd(a, b)
+
+def get_device_size(device):
+ """
+ Returns a filesystem's total size and block size in bytes.
+ For block devices, block size is the device's block size.
+ For other files (fs images), block size is 1 byte.
+
+ @param device: str
+ Absolute path to the device or filesystem image.
+ @return: tuple(int, int)
+ The filesystem's size and its block size.
+ """
+ mode = os.stat(device).st_mode
+ if stat.S_ISBLK(mode):
+ basedevice = ""
+ partition = os.path.basename(device)
+ tmp = partition
+ while len(tmp) > 0:
+ tmp = tmp[:-1]
+ if os.path.exists("/sys/block/" + tmp):
+ basedevice = tmp
+ break
+ # Get device block size
+ file = open("/sys/block/" + basedevice + "/queue/hw_sector_size")
+ blocksize = int(file.readline())
+ file.close()
+ # Get partition size
+ file = open("/sys/block/" + basedevice + "/" + partition + "/size")
+ size = int(file.readline()) * blocksize
+ file.close()
+ else:
+ size = os.path.getsize(device)
+ blocksize = 1
+
+ return size, blocksize
+
+class RawFSLowSpaceError(Exception):
+ pass
+
+class RawFSItem:
+ __slots__ = ['source', 'destination', 'filesystem', 'resize']
+
+ def copy(self, current=0, total=1):
+ """
+ Copies a raw filesystem on a disk partition, and grow it to the full destination
+ partition's size if required.
+
+ @param current: int
+ The index of the current item in the filesystems list
+ (used for progress reporting)
+ @param total: int
+ The number of items in the filesystems list
+ (used for progress reporting)
+ """
+ count = 0
+
+ libcalamares.utils.debug("Copying {} to {}".format(self.source, self.destination))
+
+ srcsize, srcblksize = get_device_size(self.source)
+ destsize, destblksize = get_device_size(self.destination)
+
+ if destsize < srcsize:
+ raise RawFSLowSpaceError
+ return
+
+ # Compute transfer block size (100x the LCM of the block sizes seems a good fit)
+ blksize = int(100 * lcm(srcblksize, destblksize))
+
+ # Execute copy
+ src = open(self.source, "rb")
+ dest = open(self.destination, "wb")
+ buffer = src.read(blksize)
+ while len(buffer) > 0:
+ dest.write(buffer)
+ count += len(buffer)
+ # Compute job progress
+ progress = ((count / srcsize) + (current)) / total
+ libcalamares.job.setprogress(progress)
+ # Read next data block
+ buffer = src.read(blksize)
+ src.close()
+ dest.close()
+
+ if self.resize:
+ if "ext" in self.filesystem:
+ libcalamares.utils.debug("Resizing filesystem on {}".format(self.destination))
+ subprocess.run(["e2fsck", "-f", "-y", self.destination])
+ subprocess.run(["resize2fs", self.destination])
+
+ def __init__(self, config, device, fs):
+ libcalamares.utils.debug("Adding an entry for raw copy of {} to {}".format(
+ config["source"], device))
+ self.source = config["source"]
+ # If source is a mount point, look for the actual device mounted on it
+ if os.path.ismount(self.source):
+ procmounts = open("/proc/mounts", "r")
+ for line in procmounts:
+ if self.source in line.split():
+ self.source = line.split()[0]
+ break
+
+ self.destination = device
+ self.filesystem = fs
+ try:
+ self.resize = bool(config["resize"])
+ except KeyError:
+ self.resize = False
+
+def update_global_storage(item, gs):
+ for partition in gs:
+ if partition["device"] == item.destination:
+ ret = subprocess.run(["blkid", "-s", "UUID", "-o", "value", item.destination],
+ capture_output=True, text=True)
+ if ret.returncode == 0:
+ libcalamares.utils.debug("Setting {} UUID to {}".format(item.destination,
+ ret.stdout.rstrip()))
+ gs[gs.index(partition)]["uuid"] = ret.stdout.rstrip()
+ libcalamares.globalstorage.remove("partitions")
+ libcalamares.globalstorage.insert("partitions", gs)
+
+def run():
+ """Raw filesystem copy module"""
+ filesystems = list()
+ partitions = libcalamares.globalstorage.value("partitions")
+
+ for partition in partitions:
+ if partition["mountPoint"]:
+ for src in libcalamares.job.configuration["targets"]:
+ if src["mountPoint"] == partition["mountPoint"]:
+ filesystems.append(RawFSItem(src, partition["device"], partition["fs"]))
+
+ for item in filesystems:
+ try:
+ item.copy(filesystems.index(item), len(filesystems))
+ except RawFSLowSpaceError:
+ return ("Not enough free space",
+ "{} partition is too small to copy {} on it".format(item.destination, item.source))
+ update_global_storage(item, partitions)
+
+ return None
diff --git a/src/modules/rawfs/module.desc b/src/modules/rawfs/module.desc
new file mode 100644
index 000000000..aaf65c183
--- /dev/null
+++ b/src/modules/rawfs/module.desc
@@ -0,0 +1,7 @@
+# Module metadata file for block-copy jobmodule
+# Syntax is YAML 1.2
+---
+type: "job"
+name: "rawfs"
+interface: "python"
+script: "main.py"
diff --git a/src/modules/rawfs/rawfs.conf b/src/modules/rawfs/rawfs.conf
new file mode 100644
index 000000000..6a314ce1b
--- /dev/null
+++ b/src/modules/rawfs/rawfs.conf
@@ -0,0 +1,24 @@
+# Configuration for the rawfs module: raw filesystem copy to a block device
+
+---
+
+# To apply a custom partition layout, it has to be defined this way :
+#
+# targets:
+# - mountPoint: /
+# source: /
+# - mountPoint: /home
+# source: /images/home.img
+# resize: true
+# - mountPoint: /data
+# source: /dev/mmcblk0p3
+#
+# For each target, the following attributes must be defined:
+# * mountPoint: The mount point of the destination device on the installed system
+# The corresponding block device will automatically be identified and used as the
+# destination for the operation
+# * source: The source filesystem; it can be the mount point of a locally (on the
+# live system) mounted filesystem, a disk image, or a block device
+# * resize (optional): Expand the destination filesystem to fill the whole
+# partition at the end of the operation; this works only with ext filesystems
+# for now