--- morphlib/builder2.py | 238 +------------------- morphlib/plugins/syslinux-disk-builder_plugin.py | 276 ++++++++++++++++++++++ without-test-modules | 1 + 3 files changed, 278 insertions(+), 237 deletions(-) create mode 100644 morphlib/plugins/syslinux-disk-builder_plugin.py diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 9bef5db..e589662 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -477,248 +477,12 @@ class SystemKindBuilder(BuilderBase): # pragma: no cover ''' -class SyslinuxDiskBuilder(SystemKindBuilder): # pragma: no cover - - system_kind = 'syslinux-disk' - - def build_and_cache(self): - with self.build_watch('overall-build'): - self.app.status(msg='Building system %(system_name)s', - system_name=self.artifact.name) - - arch = self.artifact.source.morphology['arch'] - - rootfs_artifact = self.new_artifact( - self.artifact.source.morphology['name'] + '-rootfs') - handle = self.local_artifact_cache.put(rootfs_artifact) - image_name = handle.name - - self._create_image(image_name) - self._partition_image(image_name) - self._install_mbr(arch, image_name) - partition = self._setup_device_mapping(image_name) - - mount_point = None - try: - self._create_fs(partition) - mount_point = self.staging_area.destdir(self.artifact.source) - self._mount(partition, mount_point) - factory_path = os.path.join(mount_point, 'factory') - self._create_subvolume(factory_path) - self._unpack_strata(factory_path) - self._create_fstab(factory_path) - if arch in ('x86', 'x86_64'): - self._create_extlinux_config(factory_path) - self._create_subvolume_snapshot( - mount_point, 'factory', 'factory-run') - factory_run_path = os.path.join(mount_point, 'factory-run') - self._install_boot_files(arch, factory_run_path, mount_point) - if arch in ('x86', 'x86_64'): - self._install_extlinux(mount_point) - if arch in ('arm',): - a = self.new_artifact( - self.artifact.source.morphology['name']+'-kernel') - with self.local_artifact_cache.put(a) as dest: - with open(os.path.join(factory_path, - 'boot', - 'zImage')) as kernel: - shutil.copyfileobj(kernel, dest) - - self._unmount(mount_point) - except BaseException, e: - logging.error(traceback.format_exc()) - self.app.status(msg='Error while building system', - error=True) - self._unmount(mount_point) - self._undo_device_mapping(image_name) - handle.abort() - raise - - self._undo_device_mapping(image_name) - handle.close() - - self.save_build_times() - return [self.artifact] - - def _create_image(self, image_name): - self.app.status(msg='Creating disk image %(filename)s', - filename=image_name, chatty=True) - with self.build_watch('create-image'): - morphlib.fsutils.create_image( - self.app.runcmd, image_name, - self.artifact.source.morphology['disk-size']) - - def _partition_image(self, image_name): - self.app.status(msg='Partitioning disk image %(filename)s', - filename=image_name) - with self.build_watch('partition-image'): - morphlib.fsutils.partition_image(self.app.runcmd, image_name) - - def _install_mbr(self, arch, image_name): - self.app.status(msg='Installing mbr on disk image %(filename)s', - filename=image_name, chatty=True) - if arch not in ('x86', 'x86_64'): - return - with self.build_watch('install-mbr'): - morphlib.fsutils.install_syslinux_mbr(self.app.runcmd, image_name) - - def _setup_device_mapping(self, image_name): - self.app.status(msg='Device mapping partitions in %(filename)s', - filename=image_name, chatty=True) - with self.build_watch('setup-device-mapper'): - return morphlib.fsutils.setup_device_mapping(self.app.runcmd, - image_name) - - def _create_fs(self, partition): - self.app.status(msg='Creating filesystem on %(partition)s', - partition=partition, chatty=True) - with self.build_watch('create-filesystem'): - morphlib.fsutils.create_fs(self.app.runcmd, partition) - - def _mount(self, partition, mount_point): - self.app.status(msg='Mounting %(partition)s on %(mount_point)s', - partition=partition, mount_point=mount_point, - chatty=True) - with self.build_watch('mount-filesystem'): - morphlib.fsutils.mount(self.app.runcmd, partition, mount_point) - - def _create_subvolume(self, path): - self.app.status(msg='Creating subvolume %(path)s', - path=path, chatty=True) - with self.build_watch('create-factory-subvolume'): - self.app.runcmd(['btrfs', 'subvolume', 'create', path]) - - def _unpack_strata(self, path): - self.app.status(msg='Unpacking strata to %(path)s', - path=path, chatty=True) - with self.build_watch('unpack-strata'): - # download the stratum artifacts if necessary - download_depends(self.artifact.dependencies, - self.local_artifact_cache, - self.remote_artifact_cache, - ('meta',)) - - # download the chunk artifacts if necessary - for stratum_artifact in self.artifact.dependencies: - f = self.local_artifact_cache.get(stratum_artifact) - chunks = [ArtifactCacheReference(a) for a in json.load(f)] - download_depends(chunks, - self.local_artifact_cache, - self.remote_artifact_cache) - f.close() - - # check whether the strata overlap - overlaps = get_overlaps(self.artifact, self.artifact.dependencies, - self.local_artifact_cache) - if len(overlaps) > 0: - self.app.status(msg='Overlaps in system artifact ' - '%(artifact_name)s detected', - artifact_name=self.artifact.name, - error=True) - log_overlaps(overlaps) - write_overlap_metadata(self.artifact, overlaps, - self.local_artifact_cache) - - # unpack it from the local artifact cache - for stratum_artifact in self.artifact.dependencies: - f = self.local_artifact_cache.get(stratum_artifact) - for chunk in (ArtifactCacheReference(a) for a in json.load(f)): - self.app.status(msg='Unpacking chunk %(basename)s', - basename=chunk.basename(), chatty=True) - chunk_handle = self.local_artifact_cache.get(chunk) - morphlib.bins.unpack_binary_from_file(chunk_handle, path) - chunk_handle.close() - f.close() - meta = self.local_artifact_cache.get_artifact_metadata( - stratum_artifact, 'meta') - dst = morphlib.savefile.SaveFile( - os.path.join(path, 'baserock', - '%s.meta' % stratum_artifact.name), 'w') - shutil.copyfileobj(meta, dst) - dst.close() - meta.close() - - ldconfig(self.app.runcmd, path) - - def _create_fstab(self, path): - self.app.status(msg='Creating fstab in %(path)s', - path=path, chatty=True) - with self.build_watch('create-fstab'): - fstab = os.path.join(path, 'etc', 'fstab') - if not os.path.exists(os.path.dirname(fstab)):# FIXME: should exist - os.makedirs(os.path.dirname(fstab)) - with open(fstab, 'w') as f: - f.write('proc /proc proc defaults 0 0\n') - f.write('sysfs /sys sysfs defaults 0 0\n') - f.write('/dev/sda1 / btrfs defaults,rw,noatime 0 1\n') - - def _create_extlinux_config(self, path): - self.app.status(msg='Creating extlinux.conf in %(path)s', - path=path, chatty=True) - with self.build_watch('create-extlinux-config'): - config = os.path.join(path, 'extlinux.conf') - with open(config, 'w') as f: - f.write('default linux\n') - f.write('timeout 1\n') - f.write('label linux\n') - f.write('kernel /boot/vmlinuz\n') - f.write('append root=/dev/sda1 rootflags=subvol=factory-run ' - 'init=/sbin/init rw\n') - - def _create_subvolume_snapshot(self, path, source, target): - self.app.status(msg='Creating subvolume snapshot ' - '%(source)s to %(target)s', - source=source, target=target, chatty=True) - with self.build_watch('create-runtime-snapshot'): - self.app.runcmd(['btrfs', 'subvolume', 'snapshot', source, target], - cwd=path) - - def _install_boot_files(self, arch, sourcefs, targetfs): - with self.build_watch('install-boot-files'): - if arch in ('x86', 'x86_64'): - self.app.status(msg='Installing boot files into root volume', - chatty=True) - shutil.copy2(os.path.join(sourcefs, 'extlinux.conf'), - os.path.join(targetfs, 'extlinux.conf')) - os.mkdir(os.path.join(targetfs, 'boot')) - shutil.copy2(os.path.join(sourcefs, 'boot', 'vmlinuz'), - os.path.join(targetfs, 'boot', 'vmlinuz')) - shutil.copy2(os.path.join(sourcefs, 'boot', 'System.map'), - os.path.join(targetfs, 'boot', 'System.map')) - - def _install_extlinux(self, path): - self.app.status(msg='Installing extlinux to %(path)s', - path=path, chatty=True) - with self.build_watch('install-bootloader'): - self.app.runcmd(['extlinux', '--install', path]) - - # FIXME this hack seems to be necessary to let extlinux finish - self.app.runcmd(['sync']) - time.sleep(2) - - def _unmount(self, mount_point): - with self.build_watch('unmount-filesystem'): - if mount_point is not None: - self.app.status(msg='Unmounting %(mount_point)s', - mount_point=mount_point, chatty=True) - morphlib.fsutils.unmount(self.app.runcmd, mount_point) - - def _undo_device_mapping(self, image_name): - self.app.status(msg='Undoing device mappings for %(filename)s', - filename=image_name, chatty=True) - with self.build_watch('undo-device-mapper'): - morphlib.fsutils.undo_device_mapping(self.app.runcmd, image_name) - - - class SystemKindBuilderFactory(object): # pragma: no cover '''A factory class for SystemKindBuilder objects.''' def __init__(self): - self.system_kinds = [ - SyslinuxDiskBuilder, - ] + self.system_kinds = [] def register(self, klass): self.system_kinds.append(klass) diff --git a/morphlib/plugins/syslinux-disk-builder_plugin.py b/morphlib/plugins/syslinux-disk-builder_plugin.py new file mode 100644 index 0000000..f12e328 --- /dev/null +++ b/morphlib/plugins/syslinux-disk-builder_plugin.py @@ -0,0 +1,276 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import json +import logging +import os +import shutil +import time +from collections import defaultdict +import tarfile +import traceback +import subprocess + +import cliapp + +import morphlib +from morphlib.artifactcachereference import ArtifactCacheReference +from morphlib.builder2 import (SystemKindBuilder, download_depends, + get_overlaps, log_overlaps, ldconfig, + write_overlap_metadata) + + +class SyslinuxDiskBuilder(SystemKindBuilder): # pragma: no cover + + system_kind = 'syslinux-disk' + + def build_and_cache(self): + with self.build_watch('overall-build'): + self.app.status(msg='Building system %(system_name)s', + system_name=self.artifact.name) + + arch = self.artifact.source.morphology['arch'] + + rootfs_artifact = self.new_artifact( + self.artifact.source.morphology['name'] + '-rootfs') + handle = self.local_artifact_cache.put(rootfs_artifact) + image_name = handle.name + + self._create_image(image_name) + self._partition_image(image_name) + self._install_mbr(arch, image_name) + partition = self._setup_device_mapping(image_name) + + mount_point = None + try: + self._create_fs(partition) + mount_point = self.staging_area.destdir(self.artifact.source) + self._mount(partition, mount_point) + factory_path = os.path.join(mount_point, 'factory') + self._create_subvolume(factory_path) + self._unpack_strata(factory_path) + self._create_fstab(factory_path) + if arch in ('x86', 'x86_64'): + self._create_extlinux_config(factory_path) + self._create_subvolume_snapshot( + mount_point, 'factory', 'factory-run') + factory_run_path = os.path.join(mount_point, 'factory-run') + self._install_boot_files(arch, factory_run_path, mount_point) + if arch in ('x86', 'x86_64'): + self._install_extlinux(mount_point) + if arch in ('arm',): + a = self.new_artifact( + self.artifact.source.morphology['name']+'-kernel') + with self.local_artifact_cache.put(a) as dest: + with open(os.path.join(factory_path, + 'boot', + 'zImage')) as kernel: + shutil.copyfileobj(kernel, dest) + + self._unmount(mount_point) + except BaseException, e: + logging.error(traceback.format_exc()) + self.app.status(msg='Error while building system', + error=True) + self._unmount(mount_point) + self._undo_device_mapping(image_name) + handle.abort() + raise + + self._undo_device_mapping(image_name) + handle.close() + + self.save_build_times() + return [self.artifact] + + def _create_image(self, image_name): + self.app.status(msg='Creating disk image %(filename)s', + filename=image_name, chatty=True) + with self.build_watch('create-image'): + morphlib.fsutils.create_image( + self.app.runcmd, image_name, + self.artifact.source.morphology['disk-size']) + + def _partition_image(self, image_name): + self.app.status(msg='Partitioning disk image %(filename)s', + filename=image_name) + with self.build_watch('partition-image'): + morphlib.fsutils.partition_image(self.app.runcmd, image_name) + + def _install_mbr(self, arch, image_name): + self.app.status(msg='Installing mbr on disk image %(filename)s', + filename=image_name, chatty=True) + if arch not in ('x86', 'x86_64'): + return + with self.build_watch('install-mbr'): + morphlib.fsutils.install_syslinux_mbr(self.app.runcmd, image_name) + + def _setup_device_mapping(self, image_name): + self.app.status(msg='Device mapping partitions in %(filename)s', + filename=image_name, chatty=True) + with self.build_watch('setup-device-mapper'): + return morphlib.fsutils.setup_device_mapping(self.app.runcmd, + image_name) + + def _create_fs(self, partition): + self.app.status(msg='Creating filesystem on %(partition)s', + partition=partition, chatty=True) + with self.build_watch('create-filesystem'): + morphlib.fsutils.create_fs(self.app.runcmd, partition) + + def _mount(self, partition, mount_point): + self.app.status(msg='Mounting %(partition)s on %(mount_point)s', + partition=partition, mount_point=mount_point, + chatty=True) + with self.build_watch('mount-filesystem'): + morphlib.fsutils.mount(self.app.runcmd, partition, mount_point) + + def _create_subvolume(self, path): + self.app.status(msg='Creating subvolume %(path)s', + path=path, chatty=True) + with self.build_watch('create-factory-subvolume'): + self.app.runcmd(['btrfs', 'subvolume', 'create', path]) + + def _unpack_strata(self, path): + self.app.status(msg='Unpacking strata to %(path)s', + path=path, chatty=True) + with self.build_watch('unpack-strata'): + # download the stratum artifacts if necessary + download_depends(self.artifact.dependencies, + self.local_artifact_cache, + self.remote_artifact_cache, + ('meta',)) + + # download the chunk artifacts if necessary + for stratum_artifact in self.artifact.dependencies: + f = self.local_artifact_cache.get(stratum_artifact) + chunks = [ArtifactCacheReference(a) for a in json.load(f)] + download_depends(chunks, + self.local_artifact_cache, + self.remote_artifact_cache) + f.close() + + # check whether the strata overlap + overlaps = get_overlaps(self.artifact, self.artifact.dependencies, + self.local_artifact_cache) + if len(overlaps) > 0: + self.app.status(msg='Overlaps in system artifact ' + '%(artifact_name)s detected', + artifact_name=self.artifact.name, + error=True) + log_overlaps(overlaps) + write_overlap_metadata(self.artifact, overlaps, + self.local_artifact_cache) + + # unpack it from the local artifact cache + for stratum_artifact in self.artifact.dependencies: + f = self.local_artifact_cache.get(stratum_artifact) + for chunk in (ArtifactCacheReference(a) for a in json.load(f)): + self.app.status(msg='Unpacking chunk %(basename)s', + basename=chunk.basename(), chatty=True) + chunk_handle = self.local_artifact_cache.get(chunk) + morphlib.bins.unpack_binary_from_file(chunk_handle, path) + chunk_handle.close() + f.close() + meta = self.local_artifact_cache.get_artifact_metadata( + stratum_artifact, 'meta') + dst = morphlib.savefile.SaveFile( + os.path.join(path, 'baserock', + '%s.meta' % stratum_artifact.name), 'w') + shutil.copyfileobj(meta, dst) + dst.close() + meta.close() + + ldconfig(self.app.runcmd, path) + + def _create_fstab(self, path): + self.app.status(msg='Creating fstab in %(path)s', + path=path, chatty=True) + with self.build_watch('create-fstab'): + fstab = os.path.join(path, 'etc', 'fstab') + if not os.path.exists(os.path.dirname(fstab)):# FIXME: should exist + os.makedirs(os.path.dirname(fstab)) + with open(fstab, 'w') as f: + f.write('proc /proc proc defaults 0 0\n') + f.write('sysfs /sys sysfs defaults 0 0\n') + f.write('/dev/sda1 / btrfs defaults,rw,noatime 0 1\n') + + def _create_extlinux_config(self, path): + self.app.status(msg='Creating extlinux.conf in %(path)s', + path=path, chatty=True) + with self.build_watch('create-extlinux-config'): + config = os.path.join(path, 'extlinux.conf') + with open(config, 'w') as f: + f.write('default linux\n') + f.write('timeout 1\n') + f.write('label linux\n') + f.write('kernel /boot/vmlinuz\n') + f.write('append root=/dev/sda1 rootflags=subvol=factory-run ' + 'init=/sbin/init rw\n') + + def _create_subvolume_snapshot(self, path, source, target): + self.app.status(msg='Creating subvolume snapshot ' + '%(source)s to %(target)s', + source=source, target=target, chatty=True) + with self.build_watch('create-runtime-snapshot'): + self.app.runcmd(['btrfs', 'subvolume', 'snapshot', source, target], + cwd=path) + + def _install_boot_files(self, arch, sourcefs, targetfs): + with self.build_watch('install-boot-files'): + if arch in ('x86', 'x86_64'): + self.app.status(msg='Installing boot files into root volume', + chatty=True) + shutil.copy2(os.path.join(sourcefs, 'extlinux.conf'), + os.path.join(targetfs, 'extlinux.conf')) + os.mkdir(os.path.join(targetfs, 'boot')) + shutil.copy2(os.path.join(sourcefs, 'boot', 'vmlinuz'), + os.path.join(targetfs, 'boot', 'vmlinuz')) + shutil.copy2(os.path.join(sourcefs, 'boot', 'System.map'), + os.path.join(targetfs, 'boot', 'System.map')) + + def _install_extlinux(self, path): + self.app.status(msg='Installing extlinux to %(path)s', + path=path, chatty=True) + with self.build_watch('install-bootloader'): + self.app.runcmd(['extlinux', '--install', path]) + + # FIXME this hack seems to be necessary to let extlinux finish + self.app.runcmd(['sync']) + time.sleep(2) + + def _unmount(self, mount_point): + with self.build_watch('unmount-filesystem'): + if mount_point is not None: + self.app.status(msg='Unmounting %(mount_point)s', + mount_point=mount_point, chatty=True) + morphlib.fsutils.unmount(self.app.runcmd, mount_point) + + def _undo_device_mapping(self, image_name): + self.app.status(msg='Undoing device mappings for %(filename)s', + filename=image_name, chatty=True) + with self.build_watch('undo-device-mapper'): + morphlib.fsutils.undo_device_mapping(self.app.runcmd, image_name) + + +class SyslinuxDiskBuilderPlugin(cliapp.Plugin): + + def enable(self): + self.app.system_kind_builder_factory.register(SyslinuxDiskBuilder) + + def disable(self): + pass + diff --git a/without-test-modules b/without-test-modules index 5c824c1..df63957 100644 --- a/without-test-modules +++ b/without-test-modules @@ -7,3 +7,4 @@ morphlib/fsutils.py morphlib/app.py morphlib/plugins/hello_plugin.py morphlib/plugins/graphing_plugin.py +morphlib/plugins/syslinux-disk-builder_plugin.py -- 1.7.2.5

Also, rename the syslinux-disk builder plugin file to make it clear it's a system builder plugin. --- morphlib/builder2.py | 2 + morphlib/plugins/syslinux-disk-builder_plugin.py | 279 -------------------- .../plugins/syslinux-disk-systembuilder_plugin.py | 276 +++++++++++++++++++ morphlib/plugins/tarball-systembuilder_plugin.py | 161 +++++++++++ without-test-modules | 4 +- 5 files changed, 442 insertions(+), 280 deletions(-) delete mode 100644 morphlib/plugins/syslinux-disk-builder_plugin.py create mode 100644 morphlib/plugins/syslinux-disk-systembuilder_plugin.py create mode 100644 morphlib/plugins/tarball-systembuilder_plugin.py diff --git a/morphlib/builder2.py b/morphlib/builder2.py index e589662..5362dd3 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -509,6 +509,8 @@ class SystemBuilder(BuilderBase): # pragma: no cover builder = self.app.system_kind_builder_factory.new( system_kind, self.args, self.kwargs) logging.debug('Building system with %s' % repr(builder)) + self.app.status(msg='Building system %(system_name)s', + system_name=self.artifact.name) return builder.build_and_cache() diff --git a/morphlib/plugins/syslinux-disk-builder_plugin.py b/morphlib/plugins/syslinux-disk-builder_plugin.py deleted file mode 100644 index bb5fcbc..0000000 --- a/morphlib/plugins/syslinux-disk-builder_plugin.py +++ /dev/null @@ -1,279 +0,0 @@ -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import json -import logging -import os -import shutil -import time -from collections import defaultdict -import tarfile -import traceback -import subprocess - -import cliapp - -import morphlib -from morphlib.artifactcachereference import ArtifactCacheReference -from morphlib.builder2 import (SystemKindBuilder, download_depends, - get_overlaps, log_overlaps, ldconfig, - write_overlap_metadata) - - -class SyslinuxDiskBuilder(SystemKindBuilder): # pragma: no cover - - system_kind = 'syslinux-disk' - - def build_and_cache(self): - with self.build_watch('overall-build'): - self.app.status(msg='Building system %(system_name)s', - system_name=self.artifact.name) - - arch = self.artifact.source.morphology['arch'] - - rootfs_artifact = self.new_artifact( - self.artifact.source.morphology['name'] + '-rootfs') - handle = self.local_artifact_cache.put(rootfs_artifact) - image_name = handle.name - - self._create_image(image_name) - self._partition_image(image_name) - self._install_mbr(arch, image_name) - partition = self._setup_device_mapping(image_name) - - mount_point = None - try: - self._create_fs(partition) - mount_point = self.staging_area.destdir(self.artifact.source) - self._mount(partition, mount_point) - factory_path = os.path.join(mount_point, 'factory') - self._create_subvolume(factory_path) - self._unpack_strata(factory_path) - self._create_fstab(factory_path) - if arch in ('x86', 'x86_64'): - self._create_extlinux_config(factory_path) - self._create_subvolume_snapshot( - mount_point, 'factory', 'factory-run') - factory_run_path = os.path.join(mount_point, 'factory-run') - self._install_boot_files(arch, factory_run_path, mount_point) - if arch in ('x86', 'x86_64'): - self._install_extlinux(mount_point) - if arch in ('arm',): - a = self.new_artifact( - self.artifact.source.morphology['name']+'-kernel') - with self.local_artifact_cache.put(a) as dest: - with open(os.path.join(factory_path, - 'boot', - 'zImage')) as kernel: - shutil.copyfileobj(kernel, dest) - - self._unmount(mount_point) - except BaseException, e: - logging.error(traceback.format_exc()) - self.app.status(msg='Error while building system', - error=True) - self._unmount(mount_point) - self._undo_device_mapping(image_name) - handle.abort() - raise - - self._undo_device_mapping(image_name) - handle.close() - - self.save_build_times() - return [self.artifact] - - def _create_image(self, image_name): - self.app.status(msg='Creating disk image %(filename)s', - filename=image_name, chatty=True) - with self.build_watch('create-image'): - morphlib.fsutils.create_image( - self.app.runcmd, image_name, - self.artifact.source.morphology['disk-size']) - - def _partition_image(self, image_name): - self.app.status(msg='Partitioning disk image %(filename)s', - filename=image_name) - with self.build_watch('partition-image'): - morphlib.fsutils.partition_image(self.app.runcmd, image_name) - - def _install_mbr(self, arch, image_name): - self.app.status(msg='Installing mbr on disk image %(filename)s', - filename=image_name, chatty=True) - if arch not in ('x86', 'x86_64'): - return - with self.build_watch('install-mbr'): - morphlib.fsutils.install_syslinux_mbr(self.app.runcmd, image_name) - - def _setup_device_mapping(self, image_name): - self.app.status(msg='Device mapping partitions in %(filename)s', - filename=image_name, chatty=True) - with self.build_watch('setup-device-mapper'): - return morphlib.fsutils.setup_device_mapping(self.app.runcmd, - image_name) - - def _create_fs(self, partition): - self.app.status(msg='Creating filesystem on %(partition)s', - partition=partition, chatty=True) - with self.build_watch('create-filesystem'): - morphlib.fsutils.create_fs(self.app.runcmd, partition) - - def _mount(self, partition, mount_point): - self.app.status(msg='Mounting %(partition)s on %(mount_point)s', - partition=partition, mount_point=mount_point, - chatty=True) - with self.build_watch('mount-filesystem'): - morphlib.fsutils.mount(self.app.runcmd, partition, mount_point) - - def _create_subvolume(self, path): - self.app.status(msg='Creating subvolume %(path)s', - path=path, chatty=True) - with self.build_watch('create-factory-subvolume'): - self.app.runcmd(['btrfs', 'subvolume', 'create', path]) - - def _unpack_strata(self, path): - self.app.status(msg='Unpacking strata to %(path)s', - path=path, chatty=True) - with self.build_watch('unpack-strata'): - # download the stratum artifacts if necessary - download_depends(self.artifact.dependencies, - self.local_artifact_cache, - self.remote_artifact_cache, - ('meta',)) - - # download the chunk artifacts if necessary - for stratum_artifact in self.artifact.dependencies: - f = self.local_artifact_cache.get(stratum_artifact) - chunks = [ArtifactCacheReference(a) for a in json.load(f)] - download_depends(chunks, - self.local_artifact_cache, - self.remote_artifact_cache) - f.close() - - # check whether the strata overlap - overlaps = get_overlaps(self.artifact, self.artifact.dependencies, - self.local_artifact_cache) - if len(overlaps) > 0: - self.app.status(msg='Overlaps in system artifact ' - '%(artifact_name)s detected', - artifact_name=self.artifact.name, - error=True) - log_overlaps(overlaps) - write_overlap_metadata(self.artifact, overlaps, - self.local_artifact_cache) - - # unpack it from the local artifact cache - for stratum_artifact in self.artifact.dependencies: - f = self.local_artifact_cache.get(stratum_artifact) - for chunk in (ArtifactCacheReference(a) for a in json.load(f)): - self.app.status(msg='Unpacking chunk %(basename)s', - basename=chunk.basename(), chatty=True) - chunk_handle = self.local_artifact_cache.get(chunk) - morphlib.bins.unpack_binary_from_file(chunk_handle, path) - chunk_handle.close() - f.close() - meta = self.local_artifact_cache.get_artifact_metadata( - stratum_artifact, 'meta') - dst = morphlib.savefile.SaveFile( - os.path.join(path, 'baserock', - '%s.meta' % stratum_artifact.name), 'w') - shutil.copyfileobj(meta, dst) - dst.close() - meta.close() - - ldconfig(self.app.runcmd, path) - - def _create_fstab(self, path): - self.app.status(msg='Creating fstab in %(path)s', - path=path, chatty=True) - with self.build_watch('create-fstab'): - fstab = os.path.join(path, 'etc', 'fstab') - if not os.path.exists(os.path.dirname(fstab)):# FIXME: should exist - os.makedirs(os.path.dirname(fstab)) - with open(fstab, 'w') as f: - f.write('proc /proc proc defaults 0 0\n') - f.write('sysfs /sys sysfs defaults 0 0\n') - f.write('/dev/sda1 / btrfs defaults,rw,noatime 0 1\n') - - def _create_extlinux_config(self, path): - self.app.status(msg='Creating extlinux.conf in %(path)s', - path=path, chatty=True) - with self.build_watch('create-extlinux-config'): - config = os.path.join(path, 'extlinux.conf') - with open(config, 'w') as f: - f.write('default linux\n') - f.write('timeout 1\n') - f.write('label linux\n') - f.write('kernel /boot/vmlinuz\n') - f.write('append root=/dev/sda1 rootflags=subvol=factory-run ' - 'init=/sbin/init rw\n') - - def _create_subvolume_snapshot(self, path, source, target): - self.app.status(msg='Creating subvolume snapshot ' - '%(source)s to %(target)s', - source=source, target=target, chatty=True) - with self.build_watch('create-runtime-snapshot'): - self.app.runcmd(['btrfs', 'subvolume', 'snapshot', source, target], - cwd=path) - - def _install_boot_files(self, arch, sourcefs, targetfs): - with self.build_watch('install-boot-files'): - if arch in ('x86', 'x86_64'): - self.app.status(msg='Installing boot files into root volume', - chatty=True) - shutil.copy2(os.path.join(sourcefs, 'extlinux.conf'), - os.path.join(targetfs, 'extlinux.conf')) - os.mkdir(os.path.join(targetfs, 'boot')) - shutil.copy2(os.path.join(sourcefs, 'boot', 'vmlinuz'), - os.path.join(targetfs, 'boot', 'vmlinuz')) - shutil.copy2(os.path.join(sourcefs, 'boot', 'System.map'), - os.path.join(targetfs, 'boot', 'System.map')) - - def _install_extlinux(self, path): - self.app.status(msg='Installing extlinux to %(path)s', - path=path, chatty=True) - with self.build_watch('install-bootloader'): - self.app.runcmd(['extlinux', '--install', path]) - - # FIXME this hack seems to be necessary to let extlinux finish - self.app.runcmd(['sync']) - time.sleep(2) - - def _unmount(self, mount_point): - with self.build_watch('unmount-filesystem'): - if mount_point is not None: - self.app.status(msg='Unmounting %(mount_point)s', - mount_point=mount_point, chatty=True) - morphlib.fsutils.unmount(self.app.runcmd, mount_point) - - def _undo_device_mapping(self, image_name): - self.app.status(msg='Undoing device mappings for %(filename)s', - filename=image_name, chatty=True) - with self.build_watch('undo-device-mapper'): - morphlib.fsutils.undo_device_mapping(self.app.runcmd, image_name) - - -class SyslinuxDiskBuilderPlugin(cliapp.Plugin): - - def enable(self): - # Only provide this system builder on architectures that are - # supported by syslinux. - if morphlib.util.arch() in ['x86_64', 'i386', 'i486', 'i586', 'i686']: - self.app.system_kind_builder_factory.register(SyslinuxDiskBuilder) - - def disable(self): - pass - diff --git a/morphlib/plugins/syslinux-disk-systembuilder_plugin.py b/morphlib/plugins/syslinux-disk-systembuilder_plugin.py new file mode 100644 index 0000000..d91256e --- /dev/null +++ b/morphlib/plugins/syslinux-disk-systembuilder_plugin.py @@ -0,0 +1,276 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import json +import logging +import os +import shutil +import time +from collections import defaultdict +import tarfile +import traceback +import subprocess + +import cliapp + +import morphlib +from morphlib.artifactcachereference import ArtifactCacheReference +from morphlib.builder2 import (SystemKindBuilder, download_depends, + get_overlaps, log_overlaps, ldconfig, + write_overlap_metadata) + + +class SyslinuxDiskBuilder(SystemKindBuilder): # pragma: no cover + + system_kind = 'syslinux-disk' + + def build_and_cache(self): + with self.build_watch('overall-build'): + arch = self.artifact.source.morphology['arch'] + + rootfs_artifact = self.new_artifact( + self.artifact.source.morphology['name'] + '-rootfs') + handle = self.local_artifact_cache.put(rootfs_artifact) + image_name = handle.name + + self._create_image(image_name) + self._partition_image(image_name) + self._install_mbr(arch, image_name) + partition = self._setup_device_mapping(image_name) + + mount_point = None + try: + self._create_fs(partition) + mount_point = self.staging_area.destdir(self.artifact.source) + self._mount(partition, mount_point) + factory_path = os.path.join(mount_point, 'factory') + self._create_subvolume(factory_path) + self._unpack_strata(factory_path) + self._create_fstab(factory_path) + if arch in ('x86', 'x86_64'): + self._create_extlinux_config(factory_path) + self._create_subvolume_snapshot( + mount_point, 'factory', 'factory-run') + factory_run_path = os.path.join(mount_point, 'factory-run') + self._install_boot_files(arch, factory_run_path, mount_point) + if arch in ('x86', 'x86_64'): + self._install_extlinux(mount_point) + if arch in ('arm',): + a = self.new_artifact( + self.artifact.source.morphology['name']+'-kernel') + with self.local_artifact_cache.put(a) as dest: + with open(os.path.join(factory_path, + 'boot', + 'zImage')) as kernel: + shutil.copyfileobj(kernel, dest) + + self._unmount(mount_point) + except BaseException, e: + logging.error(traceback.format_exc()) + self.app.status(msg='Error while building system', + error=True) + self._unmount(mount_point) + self._undo_device_mapping(image_name) + handle.abort() + raise + + self._undo_device_mapping(image_name) + handle.close() + + self.save_build_times() + return [self.artifact] + + def _create_image(self, image_name): + self.app.status(msg='Creating disk image %(filename)s', + filename=image_name, chatty=True) + with self.build_watch('create-image'): + morphlib.fsutils.create_image( + self.app.runcmd, image_name, + self.artifact.source.morphology['disk-size']) + + def _partition_image(self, image_name): + self.app.status(msg='Partitioning disk image %(filename)s', + filename=image_name) + with self.build_watch('partition-image'): + morphlib.fsutils.partition_image(self.app.runcmd, image_name) + + def _install_mbr(self, arch, image_name): + self.app.status(msg='Installing mbr on disk image %(filename)s', + filename=image_name, chatty=True) + if arch not in ('x86', 'x86_64'): + return + with self.build_watch('install-mbr'): + morphlib.fsutils.install_syslinux_mbr(self.app.runcmd, image_name) + + def _setup_device_mapping(self, image_name): + self.app.status(msg='Device mapping partitions in %(filename)s', + filename=image_name, chatty=True) + with self.build_watch('setup-device-mapper'): + return morphlib.fsutils.setup_device_mapping(self.app.runcmd, + image_name) + + def _create_fs(self, partition): + self.app.status(msg='Creating filesystem on %(partition)s', + partition=partition, chatty=True) + with self.build_watch('create-filesystem'): + morphlib.fsutils.create_fs(self.app.runcmd, partition) + + def _mount(self, partition, mount_point): + self.app.status(msg='Mounting %(partition)s on %(mount_point)s', + partition=partition, mount_point=mount_point, + chatty=True) + with self.build_watch('mount-filesystem'): + morphlib.fsutils.mount(self.app.runcmd, partition, mount_point) + + def _create_subvolume(self, path): + self.app.status(msg='Creating subvolume %(path)s', + path=path, chatty=True) + with self.build_watch('create-factory-subvolume'): + self.app.runcmd(['btrfs', 'subvolume', 'create', path]) + + def _unpack_strata(self, path): + self.app.status(msg='Unpacking strata to %(path)s', + path=path, chatty=True) + with self.build_watch('unpack-strata'): + # download the stratum artifacts if necessary + download_depends(self.artifact.dependencies, + self.local_artifact_cache, + self.remote_artifact_cache, + ('meta',)) + + # download the chunk artifacts if necessary + for stratum_artifact in self.artifact.dependencies: + f = self.local_artifact_cache.get(stratum_artifact) + chunks = [ArtifactCacheReference(a) for a in json.load(f)] + download_depends(chunks, + self.local_artifact_cache, + self.remote_artifact_cache) + f.close() + + # check whether the strata overlap + overlaps = get_overlaps(self.artifact, self.artifact.dependencies, + self.local_artifact_cache) + if len(overlaps) > 0: + self.app.status(msg='Overlaps in system artifact ' + '%(artifact_name)s detected', + artifact_name=self.artifact.name, + error=True) + log_overlaps(overlaps) + write_overlap_metadata(self.artifact, overlaps, + self.local_artifact_cache) + + # unpack it from the local artifact cache + for stratum_artifact in self.artifact.dependencies: + f = self.local_artifact_cache.get(stratum_artifact) + for chunk in (ArtifactCacheReference(a) for a in json.load(f)): + self.app.status(msg='Unpacking chunk %(basename)s', + basename=chunk.basename(), chatty=True) + chunk_handle = self.local_artifact_cache.get(chunk) + morphlib.bins.unpack_binary_from_file(chunk_handle, path) + chunk_handle.close() + f.close() + meta = self.local_artifact_cache.get_artifact_metadata( + stratum_artifact, 'meta') + dst = morphlib.savefile.SaveFile( + os.path.join(path, 'baserock', + '%s.meta' % stratum_artifact.name), 'w') + shutil.copyfileobj(meta, dst) + dst.close() + meta.close() + + ldconfig(self.app.runcmd, path) + + def _create_fstab(self, path): + self.app.status(msg='Creating fstab in %(path)s', + path=path, chatty=True) + with self.build_watch('create-fstab'): + fstab = os.path.join(path, 'etc', 'fstab') + if not os.path.exists(os.path.dirname(fstab)):# FIXME: should exist + os.makedirs(os.path.dirname(fstab)) + with open(fstab, 'w') as f: + f.write('proc /proc proc defaults 0 0\n') + f.write('sysfs /sys sysfs defaults 0 0\n') + f.write('/dev/sda1 / btrfs defaults,rw,noatime 0 1\n') + + def _create_extlinux_config(self, path): + self.app.status(msg='Creating extlinux.conf in %(path)s', + path=path, chatty=True) + with self.build_watch('create-extlinux-config'): + config = os.path.join(path, 'extlinux.conf') + with open(config, 'w') as f: + f.write('default linux\n') + f.write('timeout 1\n') + f.write('label linux\n') + f.write('kernel /boot/vmlinuz\n') + f.write('append root=/dev/sda1 rootflags=subvol=factory-run ' + 'init=/sbin/init rw\n') + + def _create_subvolume_snapshot(self, path, source, target): + self.app.status(msg='Creating subvolume snapshot ' + '%(source)s to %(target)s', + source=source, target=target, chatty=True) + with self.build_watch('create-runtime-snapshot'): + self.app.runcmd(['btrfs', 'subvolume', 'snapshot', source, target], + cwd=path) + + def _install_boot_files(self, arch, sourcefs, targetfs): + with self.build_watch('install-boot-files'): + if arch in ('x86', 'x86_64'): + self.app.status(msg='Installing boot files into root volume', + chatty=True) + shutil.copy2(os.path.join(sourcefs, 'extlinux.conf'), + os.path.join(targetfs, 'extlinux.conf')) + os.mkdir(os.path.join(targetfs, 'boot')) + shutil.copy2(os.path.join(sourcefs, 'boot', 'vmlinuz'), + os.path.join(targetfs, 'boot', 'vmlinuz')) + shutil.copy2(os.path.join(sourcefs, 'boot', 'System.map'), + os.path.join(targetfs, 'boot', 'System.map')) + + def _install_extlinux(self, path): + self.app.status(msg='Installing extlinux to %(path)s', + path=path, chatty=True) + with self.build_watch('install-bootloader'): + self.app.runcmd(['extlinux', '--install', path]) + + # FIXME this hack seems to be necessary to let extlinux finish + self.app.runcmd(['sync']) + time.sleep(2) + + def _unmount(self, mount_point): + with self.build_watch('unmount-filesystem'): + if mount_point is not None: + self.app.status(msg='Unmounting %(mount_point)s', + mount_point=mount_point, chatty=True) + morphlib.fsutils.unmount(self.app.runcmd, mount_point) + + def _undo_device_mapping(self, image_name): + self.app.status(msg='Undoing device mappings for %(filename)s', + filename=image_name, chatty=True) + with self.build_watch('undo-device-mapper'): + morphlib.fsutils.undo_device_mapping(self.app.runcmd, image_name) + + +class SyslinuxDiskBuilderPlugin(cliapp.Plugin): + + def enable(self): + # Only provide this system builder on architectures that are + # supported by syslinux. + if morphlib.util.arch() in ['x86_64', 'i386', 'i486', 'i586', 'i686']: + self.app.system_kind_builder_factory.register(SyslinuxDiskBuilder) + + def disable(self): + pass + diff --git a/morphlib/plugins/tarball-systembuilder_plugin.py b/morphlib/plugins/tarball-systembuilder_plugin.py new file mode 100644 index 0000000..5e5e26d --- /dev/null +++ b/morphlib/plugins/tarball-systembuilder_plugin.py @@ -0,0 +1,161 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import json +import logging +import os +import shutil +import time +from collections import defaultdict +import tarfile +import traceback +import subprocess + +import cliapp + +import morphlib +from morphlib.artifactcachereference import ArtifactCacheReference +from morphlib.builder2 import (SystemKindBuilder, download_depends, + get_overlaps, log_overlaps, ldconfig, + write_overlap_metadata) + + +class RootfsTarballBuilder(SystemKindBuilder): # pragma: no cover + + system_kind = 'rootfs-tarball' + + def build_and_cache(self): + with self.build_watch('overall-build'): + arch = self.artifact.source.morphology['arch'] + + rootfs_artifact = self.new_artifact( + self.artifact.source.morphology['name'] + '-rootfs') + handle = self.local_artifact_cache.put(rootfs_artifact) + image_name = handle.name + + try: + mount_point = self.staging_area.destdir(self.artifact.source) + factory_path = mount_point + self._unpack_strata(factory_path) + self._create_fstab(factory_path) + if arch in ('x86', 'x86_64'): + self._create_extlinux_config(factory_path) + if arch in ('arm',): + a = self.new_artifact( + self.artifact.source.morphology['name']+'-kernel') + with self.local_artifact_cache.put(a) as dest: + with open(os.path.join(factory_path, + 'boot', + 'zImage')) as kernel: + shutil.copyfileobj(kernel, dest) + except BaseException, e: + logging.error(traceback.format_exc()) + self.app.status(msg='Error while building system', + error=True) + handle.abort() + raise + + handle.close() + + self.save_build_times() + return [self.artifact] + + def _unpack_strata(self, path): + self.app.status(msg='Unpacking strata to %(path)s', + path=path, chatty=True) + with self.build_watch('unpack-strata'): + # download the stratum artifacts if necessary + download_depends(self.artifact.dependencies, + self.local_artifact_cache, + self.remote_artifact_cache, + ('meta',)) + + # download the chunk artifacts if necessary + for stratum_artifact in self.artifact.dependencies: + f = self.local_artifact_cache.get(stratum_artifact) + chunks = [ArtifactCacheReference(a) for a in json.load(f)] + download_depends(chunks, + self.local_artifact_cache, + self.remote_artifact_cache) + f.close() + + # check whether the strata overlap + overlaps = get_overlaps(self.artifact, self.artifact.dependencies, + self.local_artifact_cache) + if len(overlaps) > 0: + self.app.status(msg='Overlaps in system artifact ' + '%(artifact_name)s detected', + artifact_name=self.artifact.name, + error=True) + log_overlaps(overlaps) + write_overlap_metadata(self.artifact, overlaps, + self.local_artifact_cache) + + # unpack it from the local artifact cache + for stratum_artifact in self.artifact.dependencies: + f = self.local_artifact_cache.get(stratum_artifact) + for chunk in (ArtifactCacheReference(a) for a in json.load(f)): + self.app.status(msg='Unpacking chunk %(basename)s', + basename=chunk.basename(), chatty=True) + chunk_handle = self.local_artifact_cache.get(chunk) + morphlib.bins.unpack_binary_from_file(chunk_handle, path) + chunk_handle.close() + f.close() + meta = self.local_artifact_cache.get_artifact_metadata( + stratum_artifact, 'meta') + dst = morphlib.savefile.SaveFile( + os.path.join(path, 'baserock', + '%s.meta' % stratum_artifact.name), 'w') + shutil.copyfileobj(meta, dst) + dst.close() + meta.close() + + ldconfig(self.app.runcmd, path) + + def _create_fstab(self, path): + self.app.status(msg='Creating fstab in %(path)s', + path=path, chatty=True) + with self.build_watch('create-fstab'): + fstab = os.path.join(path, 'etc', 'fstab') + if not os.path.exists(os.path.dirname(fstab)):# FIXME: should exist + os.makedirs(os.path.dirname(fstab)) + with open(fstab, 'w') as f: + f.write('proc /proc proc defaults 0 0\n') + f.write('sysfs /sys sysfs defaults 0 0\n') + f.write('/dev/sda1 / btrfs defaults,rw,noatime 0 1\n') + + def _create_extlinux_config(self, path): + self.app.status(msg='Creating extlinux.conf in %(path)s', + path=path, chatty=True) + with self.build_watch('create-extlinux-config'): + config = os.path.join(path, 'extlinux.conf') + with open(config, 'w') as f: + f.write('default linux\n') + f.write('timeout 1\n') + f.write('label linux\n') + f.write('kernel /boot/vmlinuz\n') + f.write('append root=/dev/sda1 rootflags=subvol=factory-run ' + 'init=/sbin/init rw\n') + + +class RootfsTarballBuilderPlugin(cliapp.Plugin): + + def enable(self): + self.app.system_kind_builder_factory.register(RootfsTarballBuilder) + + def disable(self): + pass + diff --git a/without-test-modules b/without-test-modules index df63957..47c5d7c 100644 --- a/without-test-modules +++ b/without-test-modules @@ -7,4 +7,6 @@ morphlib/fsutils.py morphlib/app.py morphlib/plugins/hello_plugin.py morphlib/plugins/graphing_plugin.py -morphlib/plugins/syslinux-disk-builder_plugin.py +morphlib/plugins/syslinux-disk-systembuilder_plugin.py +morphlib/plugins/tarball-systembuilder_plugin.py + -- 1.7.2.5

--- .../syslinux-disk-builds-rootfs-and-kernel.script | 94 ++++++++++++++++++++ 1 files changed, 94 insertions(+), 0 deletions(-) create mode 100755 tests.as-root/syslinux-disk-builds-rootfs-and-kernel.script diff --git a/tests.as-root/syslinux-disk-builds-rootfs-and-kernel.script b/tests.as-root/syslinux-disk-builds-rootfs-and-kernel.script new file mode 100755 index 0000000..10b563a --- /dev/null +++ b/tests.as-root/syslinux-disk-builds-rootfs-and-kernel.script @@ -0,0 +1,94 @@ +#!/bin/sh +# +# A system kind of syslinux-disk should create the disk image, plus a kernel +# file. +# +# Copyright (C) 2011, 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +set -eu + +cache="$DATADIR/cache/artifacts" +kernelrepo="$DATADIR/kernel-repo" +morphsrepo="$DATADIR/morphs-repo" +log="$DATADIR/morph.log" + +cd "$morphsrepo" +git checkout --quiet -b custom master +cat <<EOF >system.morph +{ + "name": "x86_64-system", + "kind": "system", + "system-kind": "syslinux-disk", + "arch": "x86_64", + "disk-size": "1G", + "strata": [ + "stratum" + ] +} +EOF +cat <<EOF >stratum.morph +{ + "name": "stratum", + "kind": "stratum", + "sources": [ + { + "name": "hello", + "repo": "test:chunk-repo", + "ref": "farrokh", + "build-depends": [] + }, + { + "name": "linux", + "repo": "test:kernel-repo", + "ref": "custom", + "build-depends": ["hello"] + } + ] +} +EOF +git add system.morph stratum.morph +git commit --quiet -m "foo" + +cd "$kernelrepo" +git checkout --quiet -b custom master +cat <<EOF >linux.morph +{ + "name": "linux", + "kind": "chunk", + "install-commands": [ + "mkdir -p \"\$DESTDIR/boot\"", + "touch \"\$DESTDIR\"/boot/vmlinuz", + "touch \"\$DESTDIR\"/boot/System.map" + ] +} +EOF +git add linux.morph + +git commit --quiet -m 'Make the kernel create a dummy zImage' + +"$SRCDIR/scripts/test-morph" \ + build test:morphs-repo custom system.morph > /dev/null + +for suffix in kernel rootfs +do + if [ ! -e "$cache"/*system.system-"$suffix" ]; + then + echo "No .system.system-$suffix" 1>&2 + find "$cache" 1>&2 + exit 1 + fi +done + -- 1.7.2.5

--- tests.as-root/make-patch.script | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/tests.as-root/make-patch.script b/tests.as-root/make-patch.script index 9a159a4..bd14c5a 100755 --- a/tests.as-root/make-patch.script +++ b/tests.as-root/make-patch.script @@ -45,7 +45,7 @@ EOF # Build first image. Remember the stratum. "$SRCDIR/scripts/test-morph" build test:morphs-repo master hello-system.morph -img1=$(find "$DATADIR/cache/artifacts" -maxdepth 1 -name '*.system.*') +img1=$(find "$DATADIR/cache/artifacts" -maxdepth 1 -name '*.system.*-rootfs') # Modify the chunk, in a new branch. "$SRCDIR/scripts/run-git-in" "$DATADIR/chunk-repo" checkout --quiet farrokh -- 1.7.2.5

--- morphlib/builder2.py | 54 ++++++++++++++++++++ .../plugins/syslinux-disk-systembuilder_plugin.py | 54 +------------------- morphlib/plugins/tarball-systembuilder_plugin.py | 54 +------------------- 3 files changed, 56 insertions(+), 106 deletions(-) diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 5362dd3..dd3c530 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -476,6 +476,60 @@ class SystemKindBuilder(BuilderBase): # pragma: no cover ''' + def unpack_strata(self, path): + '''Unpack strata into a directory.''' + + self.app.status(msg='Unpacking strata to %(path)s', + path=path, chatty=True) + with self.build_watch('unpack-strata'): + # download the stratum artifacts if necessary + download_depends(self.artifact.dependencies, + self.local_artifact_cache, + self.remote_artifact_cache, + ('meta',)) + + # download the chunk artifacts if necessary + for stratum_artifact in self.artifact.dependencies: + f = self.local_artifact_cache.get(stratum_artifact) + chunks = [ArtifactCacheReference(a) for a in json.load(f)] + download_depends(chunks, + self.local_artifact_cache, + self.remote_artifact_cache) + f.close() + + # check whether the strata overlap + overlaps = get_overlaps(self.artifact, self.artifact.dependencies, + self.local_artifact_cache) + if len(overlaps) > 0: + self.app.status(msg='Overlaps in system artifact ' + '%(artifact_name)s detected', + artifact_name=self.artifact.name, + error=True) + log_overlaps(overlaps) + write_overlap_metadata(self.artifact, overlaps, + self.local_artifact_cache) + + # unpack it from the local artifact cache + for stratum_artifact in self.artifact.dependencies: + f = self.local_artifact_cache.get(stratum_artifact) + for chunk in (ArtifactCacheReference(a) for a in json.load(f)): + self.app.status(msg='Unpacking chunk %(basename)s', + basename=chunk.basename(), chatty=True) + chunk_handle = self.local_artifact_cache.get(chunk) + morphlib.bins.unpack_binary_from_file(chunk_handle, path) + chunk_handle.close() + f.close() + meta = self.local_artifact_cache.get_artifact_metadata( + stratum_artifact, 'meta') + dst = morphlib.savefile.SaveFile( + os.path.join(path, 'baserock', + '%s.meta' % stratum_artifact.name), 'w') + shutil.copyfileobj(meta, dst) + dst.close() + meta.close() + + ldconfig(self.app.runcmd, path) + class SystemKindBuilderFactory(object): # pragma: no cover diff --git a/morphlib/plugins/syslinux-disk-systembuilder_plugin.py b/morphlib/plugins/syslinux-disk-systembuilder_plugin.py index 7d46d61..41a9500 100644 --- a/morphlib/plugins/syslinux-disk-systembuilder_plugin.py +++ b/morphlib/plugins/syslinux-disk-systembuilder_plugin.py @@ -58,7 +58,7 @@ class SyslinuxDiskBuilder(SystemKindBuilder): # pragma: no cover self._mount(partition, mount_point) factory_path = os.path.join(mount_point, 'factory') self._create_subvolume(factory_path) - self._unpack_strata(factory_path) + self.unpack_strata(factory_path) self._create_fstab(factory_path) self._create_extlinux_config(factory_path) self._create_subvolume_snapshot( @@ -139,58 +139,6 @@ class SyslinuxDiskBuilder(SystemKindBuilder): # pragma: no cover with self.build_watch('create-factory-subvolume'): self.app.runcmd(['btrfs', 'subvolume', 'create', path]) - def _unpack_strata(self, path): - self.app.status(msg='Unpacking strata to %(path)s', - path=path, chatty=True) - with self.build_watch('unpack-strata'): - # download the stratum artifacts if necessary - download_depends(self.artifact.dependencies, - self.local_artifact_cache, - self.remote_artifact_cache, - ('meta',)) - - # download the chunk artifacts if necessary - for stratum_artifact in self.artifact.dependencies: - f = self.local_artifact_cache.get(stratum_artifact) - chunks = [ArtifactCacheReference(a) for a in json.load(f)] - download_depends(chunks, - self.local_artifact_cache, - self.remote_artifact_cache) - f.close() - - # check whether the strata overlap - overlaps = get_overlaps(self.artifact, self.artifact.dependencies, - self.local_artifact_cache) - if len(overlaps) > 0: - self.app.status(msg='Overlaps in system artifact ' - '%(artifact_name)s detected', - artifact_name=self.artifact.name, - error=True) - log_overlaps(overlaps) - write_overlap_metadata(self.artifact, overlaps, - self.local_artifact_cache) - - # unpack it from the local artifact cache - for stratum_artifact in self.artifact.dependencies: - f = self.local_artifact_cache.get(stratum_artifact) - for chunk in (ArtifactCacheReference(a) for a in json.load(f)): - self.app.status(msg='Unpacking chunk %(basename)s', - basename=chunk.basename(), chatty=True) - chunk_handle = self.local_artifact_cache.get(chunk) - morphlib.bins.unpack_binary_from_file(chunk_handle, path) - chunk_handle.close() - f.close() - meta = self.local_artifact_cache.get_artifact_metadata( - stratum_artifact, 'meta') - dst = morphlib.savefile.SaveFile( - os.path.join(path, 'baserock', - '%s.meta' % stratum_artifact.name), 'w') - shutil.copyfileobj(meta, dst) - dst.close() - meta.close() - - ldconfig(self.app.runcmd, path) - def _create_fstab(self, path): self.app.status(msg='Creating fstab in %(path)s', path=path, chatty=True) diff --git a/morphlib/plugins/tarball-systembuilder_plugin.py b/morphlib/plugins/tarball-systembuilder_plugin.py index 5e5e26d..5582eec 100644 --- a/morphlib/plugins/tarball-systembuilder_plugin.py +++ b/morphlib/plugins/tarball-systembuilder_plugin.py @@ -49,7 +49,7 @@ class RootfsTarballBuilder(SystemKindBuilder): # pragma: no cover try: mount_point = self.staging_area.destdir(self.artifact.source) factory_path = mount_point - self._unpack_strata(factory_path) + self.unpack_strata(factory_path) self._create_fstab(factory_path) if arch in ('x86', 'x86_64'): self._create_extlinux_config(factory_path) @@ -73,58 +73,6 @@ class RootfsTarballBuilder(SystemKindBuilder): # pragma: no cover self.save_build_times() return [self.artifact] - def _unpack_strata(self, path): - self.app.status(msg='Unpacking strata to %(path)s', - path=path, chatty=True) - with self.build_watch('unpack-strata'): - # download the stratum artifacts if necessary - download_depends(self.artifact.dependencies, - self.local_artifact_cache, - self.remote_artifact_cache, - ('meta',)) - - # download the chunk artifacts if necessary - for stratum_artifact in self.artifact.dependencies: - f = self.local_artifact_cache.get(stratum_artifact) - chunks = [ArtifactCacheReference(a) for a in json.load(f)] - download_depends(chunks, - self.local_artifact_cache, - self.remote_artifact_cache) - f.close() - - # check whether the strata overlap - overlaps = get_overlaps(self.artifact, self.artifact.dependencies, - self.local_artifact_cache) - if len(overlaps) > 0: - self.app.status(msg='Overlaps in system artifact ' - '%(artifact_name)s detected', - artifact_name=self.artifact.name, - error=True) - log_overlaps(overlaps) - write_overlap_metadata(self.artifact, overlaps, - self.local_artifact_cache) - - # unpack it from the local artifact cache - for stratum_artifact in self.artifact.dependencies: - f = self.local_artifact_cache.get(stratum_artifact) - for chunk in (ArtifactCacheReference(a) for a in json.load(f)): - self.app.status(msg='Unpacking chunk %(basename)s', - basename=chunk.basename(), chatty=True) - chunk_handle = self.local_artifact_cache.get(chunk) - morphlib.bins.unpack_binary_from_file(chunk_handle, path) - chunk_handle.close() - f.close() - meta = self.local_artifact_cache.get_artifact_metadata( - stratum_artifact, 'meta') - dst = morphlib.savefile.SaveFile( - os.path.join(path, 'baserock', - '%s.meta' % stratum_artifact.name), 'w') - shutil.copyfileobj(meta, dst) - dst.close() - meta.close() - - ldconfig(self.app.runcmd, path) - def _create_fstab(self, path): self.app.status(msg='Creating fstab in %(path)s', path=path, chatty=True) -- 1.7.2.5

If you want extlinux, use a system-kind that adds it. Just having the config file without actually installing extlinux seems less than useful. --- morphlib/plugins/tarball-systembuilder_plugin.py | 15 --------------- 1 files changed, 0 insertions(+), 15 deletions(-) diff --git a/morphlib/plugins/tarball-systembuilder_plugin.py b/morphlib/plugins/tarball-systembuilder_plugin.py index 9b3f11f..ec81a91 100644 --- a/morphlib/plugins/tarball-systembuilder_plugin.py +++ b/morphlib/plugins/tarball-systembuilder_plugin.py @@ -51,8 +51,6 @@ class RootfsTarballBuilder(SystemKindBuilder): # pragma: no cover factory_path = mount_point self.unpack_strata(factory_path) self.create_fstab(factory_path) - if arch in ('x86', 'x86_64'): - self._create_extlinux_config(factory_path) if arch in ('arm',): a = self.new_artifact( self.artifact.source.morphology['name']+'-kernel') @@ -72,19 +70,6 @@ class RootfsTarballBuilder(SystemKindBuilder): # pragma: no cover self.save_build_times() return [self.artifact] - - def _create_extlinux_config(self, path): - self.app.status(msg='Creating extlinux.conf in %(path)s', - path=path, chatty=True) - with self.build_watch('create-extlinux-config'): - config = os.path.join(path, 'extlinux.conf') - with open(config, 'w') as f: - f.write('default linux\n') - f.write('timeout 1\n') - f.write('label linux\n') - f.write('kernel /boot/vmlinuz\n') - f.write('append root=/dev/sda1 rootflags=subvol=factory-run ' - 'init=/sbin/init rw\n') class RootfsTarballBuilderPlugin(cliapp.Plugin): -- 1.7.2.5

--- morphlib/builder2.py | 19 +++++++++++++++++++ .../plugins/syslinux-disk-systembuilder_plugin.py | 10 +--------- morphlib/plugins/tarball-systembuilder_plugin.py | 8 +------- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 83f5ae8..5820dc4 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -550,6 +550,25 @@ class SystemKindBuilder(BuilderBase): # pragma: no cover f.write('sysfs /sys sysfs defaults 0 0\n') f.write('/dev/sda1 / btrfs defaults,rw,noatime 0 1\n') + def copy_kernel_into_artifact_cache(self, path): + '''Copy the installed kernel image into the local artifact cache. + + The kernel image will be a separate artifact from the root + filesystem/disk image/whatever. This is sometimes useful with + funky bootloaders or virtualisation. + + ''' + + name = self.artifact.source.morphology['name']+'-kernel' + a = self.new_artifact(name) + with self.local_artifact_cache.put(a) as dest: + for basename in ['zImage', 'vmlinuz']: + installed_path = os.path.join(path, 'boot', basename) + if os.path.exists(installed_path): + with open(installed_path) as kernel: + shutil.copyfileobj(kernel, dest) + break + class SystemKindBuilderFactory(object): # pragma: no cover diff --git a/morphlib/plugins/syslinux-disk-systembuilder_plugin.py b/morphlib/plugins/syslinux-disk-systembuilder_plugin.py index 3728f55..4c6f25a 100644 --- a/morphlib/plugins/syslinux-disk-systembuilder_plugin.py +++ b/morphlib/plugins/syslinux-disk-systembuilder_plugin.py @@ -66,15 +66,7 @@ class SyslinuxDiskBuilder(SystemKindBuilder): # pragma: no cover factory_run_path = os.path.join(mount_point, 'factory-run') self._install_boot_files(arch, factory_run_path, mount_point) self._install_extlinux(mount_point) - - a = self.new_artifact( - self.artifact.source.morphology['name']+'-kernel') - with self.local_artifact_cache.put(a) as dest: - with open(os.path.join(factory_path, - 'boot', - 'vmlinuz')) as kernel: - shutil.copyfileobj(kernel, dest) - + self.copy_kernel_into_artifact_cache(factory_path) self._unmount(mount_point) except BaseException, e: logging.error(traceback.format_exc()) diff --git a/morphlib/plugins/tarball-systembuilder_plugin.py b/morphlib/plugins/tarball-systembuilder_plugin.py index fecf30a..5975a5c 100644 --- a/morphlib/plugins/tarball-systembuilder_plugin.py +++ b/morphlib/plugins/tarball-systembuilder_plugin.py @@ -51,13 +51,7 @@ class RootfsTarballBuilder(SystemKindBuilder): # pragma: no cover factory_path = mount_point self.unpack_strata(factory_path) self.create_fstab(factory_path) - a = self.new_artifact( - self.artifact.source.morphology['name']+'-kernel') - with self.local_artifact_cache.put(a) as dest: - with open(os.path.join(factory_path, - 'boot', - 'zImage')) as kernel: - shutil.copyfileobj(kernel, dest) + self.copy_kernel_into_artifact_cache(factory_path) except BaseException, e: logging.error(traceback.format_exc()) self.app.status(msg='Error while building system', -- 1.7.2.5

--- morphlib/plugins/tarball-systembuilder_plugin.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/morphlib/plugins/tarball-systembuilder_plugin.py b/morphlib/plugins/tarball-systembuilder_plugin.py index 1114aaf..8afe6bc 100644 --- a/morphlib/plugins/tarball-systembuilder_plugin.py +++ b/morphlib/plugins/tarball-systembuilder_plugin.py @@ -41,8 +41,8 @@ class RootfsTarballBuilder(SystemKindBuilder): # pragma: no cover with self.build_watch('overall-build'): arch = self.artifact.source.morphology['arch'] - rootfs_artifact = self.new_artifact( - self.artifact.source.morphology['name'] + '-rootfs') + rootfs_name = self.artifact.source.morphology['name'] + '-rootfs' + rootfs_artifact = self.new_artifact(rootfs_name) handle = self.local_artifact_cache.put(rootfs_artifact) try: -- 1.7.2.5

--- .../rootfs-tarball-builds-rootfs-and-kernel.script | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/tests.as-root/rootfs-tarball-builds-rootfs-and-kernel.script b/tests.as-root/rootfs-tarball-builds-rootfs-and-kernel.script index e88e771..3005574 100755 --- a/tests.as-root/rootfs-tarball-builds-rootfs-and-kernel.script +++ b/tests.as-root/rootfs-tarball-builds-rootfs-and-kernel.script @@ -34,7 +34,7 @@ cat <<EOF >system.morph "name": "system", "kind": "system", "system-kind": "rootfs-tarball", - "arch": "$(arch)", + "arch": "$arch", "disk-size": "1G", "strata": [ "stratum" -- 1.7.2.5