We checksum each file individually so that we can later verify that there has been no data corruption in the underlying btrfs filesystem on a per-file basis. The result is stored in /baserock/checksums.txt inside the image. --- morphlib/builder2.py | 15 ++++++++++++++- 1 files changed, 14 insertions(+), 1 deletions(-) diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 03a6214..91fa8a6 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -492,6 +492,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover self._create_fstab(factory_path) if arch in ('x86', 'x86_64'): self._create_extlinux_config(factory_path) + self._create_checksums(factory_path) self._create_subvolume_snapshot( mount_point, 'factory', 'factory-run') factory_run_path = os.path.join(mount_point, 'factory-run') @@ -506,7 +507,6 @@ class SystemBuilder(BuilderBase): # pragma: no cover 'boot', 'zImage')) as kernel: shutil.copyfileobj(kernel, dest) - self._unmount(mount_point) except BaseException, e: logging.error(traceback.format_exc()) @@ -648,6 +648,19 @@ class SystemBuilder(BuilderBase): # pragma: no cover f.write('append root=/dev/sda1 rootflags=subvol=factory-run ' 'init=/sbin/init rw\n') + def _create_checksums(self, root): + # these are used to verify that there has been no btrfs corruption + self.app.status(msg='Calculating checksums for rootfs contents', + chatty=True) + with self.build_watch('checksum-rootfs'): + checksums = os.path.join(root, 'baserock', 'checksums.txt') + with open(checksums, 'w') as f: + self.app.runcmd(['find', '.', + '-type', 'f', + '!', '-path', './baserock/checksums.txt', + '-exec', 'md5sum', '{}', '+'], + cwd=root, stdout=f) + def _create_subvolume_snapshot(self, path, source, target): self.app.status(msg='Creating subvolume snapshot ' '%(source)s to %(target)s', -- 1.7.7.6

We checksum each file individually so that we can later verify that there has been no data corruption in the underlying btrfs filesystem on a per-file basis. The result is stored in /baserock/checksums.md5 inside the image. --- morphlib/builder2.py | 17 ++++++++++++++++- 1 files changed, 16 insertions(+), 1 deletions(-) diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 03a6214..be980b0 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -492,6 +492,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover self._create_fstab(factory_path) if arch in ('x86', 'x86_64'): self._create_extlinux_config(factory_path) + self._create_checksums(factory_path) self._create_subvolume_snapshot( mount_point, 'factory', 'factory-run') factory_run_path = os.path.join(mount_point, 'factory-run') @@ -506,7 +507,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover 'boot', 'zImage')) as kernel: shutil.copyfileobj(kernel, dest) - + self._unmount(mount_point) except BaseException, e: logging.error(traceback.format_exc()) @@ -648,6 +649,20 @@ class SystemBuilder(BuilderBase): # pragma: no cover f.write('append root=/dev/sda1 rootflags=subvol=factory-run ' 'init=/sbin/init rw\n') + def _create_checksums(self, root): + '''These are used to verify that there has been no btrfs corruption.''' + + self.app.status(msg='Calculating checksums for rootfs contents', + chatty=True) + with self.build_watch('checksum-rootfs'): + checksums = os.path.join(root, 'baserock', 'checksums.md5') + with open(checksums, 'w') as f: + self.app.runcmd(['find', '.', + '-type', 'f', + '!', '-path', './baserock/checksums.md5', + '-exec', 'md5sum', '{}', '+'], + cwd=root, stdout=f) + def _create_subvolume_snapshot(self, path, source, target): self.app.status(msg='Creating subvolume snapshot ' '%(source)s to %(target)s', -- 1.7.7.6

Previously it would extract the stratum tarballs, since it was easier than mounting the disk image. Then when strata became chunk lists, this created the tarball, then extracted it. Now systems may contain files that aren't in the component strata, this causes problems for tbdiff-deploy, since it tries to alter a file that isn't there. So to fix this, it mounts the disk image, like it should have from the beginning. --- tests.as-root/make-patch.script | 28 ++++++++++++++++++++++------ 1 files changed, 22 insertions(+), 6 deletions(-) diff --git a/tests.as-root/make-patch.script b/tests.as-root/make-patch.script index b5385fc..efebdc0 100755 --- a/tests.as-root/make-patch.script +++ b/tests.as-root/make-patch.script @@ -46,7 +46,6 @@ EOF # Build first image. Remember the stratum. "$SRCDIR/scripts/test-morph" build test:morphs-repo master hello-system.morph img1=$(find "$DATADIR/cache/artifacts" -maxdepth 1 -name '*.system.*') -stratum1=$(find "$DATADIR/cache/artifacts" -maxdepth 1 -name '*.hello-stratum') # Modify the chunk, in a new branch. "$SRCDIR/scripts/run-git-in" "$DATADIR/chunk-repo" checkout --quiet farrokh @@ -73,12 +72,28 @@ patch="$DATADIR/patch" test:morphs-repo master hello-system.morph \ test:morphs-repo alfred hello-system.morph -# Unpack the original stratum and run the program. +# find out the offset of the partition in the disk image +# the gratuitous syncing gives enough time for the device to appear +loop=$(losetup -f --show "$img1") +sync +offset=$(sfdisk -d "$loop" | + sed -n '/start=/s/^.*start=\s*\([0-9]\+\)[^0-9].*$/\1/p' | + head -n 1) +ssz=`cat "/sys/devices/virtual/block/$(basename "$loop")/queue/hw_sector_size"` +losetup -d "$loop" +sync + +# mount the disk mkdir "$DATADIR/unpacked" +# this has to be done by losetup then mount, because busybox mount doesn't +# accept the -o offset= option +loop=$(losetup -o "$(expr "$offset" '*' "$ssz")" -f --show "$img1") +sync +trap 'losetup -d "$loop"' INT TERM EXIT +mount -o subvol=factory-run -t btrfs "$loop" "$DATADIR/unpacked" +trap 'umount "$DATADIR/unpacked"; sync; losetup -d "$loop"' INT TERM EXIT + cd "$DATADIR/unpacked" -"$SRCDIR/scripts/assemble-stratum" --cachedir "$DATADIR/cache" "$stratum1" \ - "$DATADIR/stratum.tar" hello-stratum -tar -xf "$DATADIR/stratum.tar" echo "old version:" ./bin/hello @@ -89,4 +104,5 @@ echo "new version:" # Done. echo "Done." - +# chdir out of mounted folder, so it can be unmounted +cd / -- 1.7.5.4

Previously it would extract the stratum tarballs, since it was easier than mounting the disk image. Then when strata became chunk lists, this created the tarball, then extracted it. Now systems may contain files that aren't in the component strata, this causes problems for tbdiff-deploy, since it tries to alter a file that isn't there. So to fix this, it mounts the disk image, like it should have from the beginning. --- tests.as-root/make-patch.script | 28 ++++++++++++++++++++++------ 1 files changed, 22 insertions(+), 6 deletions(-) diff --git a/tests.as-root/make-patch.script b/tests.as-root/make-patch.script index b5385fc..9a159a4 100755 --- a/tests.as-root/make-patch.script +++ b/tests.as-root/make-patch.script @@ -46,7 +46,6 @@ EOF # Build first image. Remember the stratum. "$SRCDIR/scripts/test-morph" build test:morphs-repo master hello-system.morph img1=$(find "$DATADIR/cache/artifacts" -maxdepth 1 -name '*.system.*') -stratum1=$(find "$DATADIR/cache/artifacts" -maxdepth 1 -name '*.hello-stratum') # Modify the chunk, in a new branch. "$SRCDIR/scripts/run-git-in" "$DATADIR/chunk-repo" checkout --quiet farrokh @@ -73,12 +72,28 @@ patch="$DATADIR/patch" test:morphs-repo master hello-system.morph \ test:morphs-repo alfred hello-system.morph -# Unpack the original stratum and run the program. +# find out the offset of the partition in the disk image +# udevadm settle is used to wait until devices have appeared +loop=$(losetup -f --show "$img1") +udevadm settle +offset=$(sfdisk -d "$loop" | + sed -n '/start=/s/^.*start=\s*\([0-9]\+\)[^0-9].*$/\1/p' | + head -n 1) +ssz=`cat "/sys/devices/virtual/block/$(basename "$loop")/queue/hw_sector_size"` +losetup -d "$loop" +udevadm settle + +# mount the disk mkdir "$DATADIR/unpacked" +# this has to be done by losetup then mount, because busybox mount doesn't +# accept the -o offset= option +loop=$(losetup -o "$(expr "$offset" '*' "$ssz")" -f --show "$img1") +udevadm settle +trap 'losetup -d "$loop"' INT TERM EXIT +mount -o subvol=factory-run -t btrfs "$loop" "$DATADIR/unpacked" +trap 'umount "$DATADIR/unpacked"; losetup -d "$loop"' INT TERM EXIT + cd "$DATADIR/unpacked" -"$SRCDIR/scripts/assemble-stratum" --cachedir "$DATADIR/cache" "$stratum1" \ - "$DATADIR/stratum.tar" hello-stratum -tar -xf "$DATADIR/stratum.tar" echo "old version:" ./bin/hello @@ -89,4 +104,5 @@ echo "new version:" # Done. echo "Done." - +# chdir out of mounted folder, so it can be unmounted +cd / -- 1.7.5.4