This was omittted from the MorphologyLoader due to cluster morphologies being added at about the same time. This bug escaped detection since the MorphologyLoader was not required to deploy. It soon will be. --- morphlib/morphloader.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index 702a330..6f50fd5 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -332,7 +332,9 @@ class MorphologyLoader(object): if key not in morphology: morphology[key] = defaults[key] - if kind == 'system': + if kind == 'cluster': + self._set_cluster_defaults(morphology) + elif kind == 'system': self._set_system_defaults(morphology) elif kind == 'stratum': self._set_stratum_defaults(morphology) @@ -355,6 +357,13 @@ class MorphologyLoader(object): if kind == 'stratum': self._unset_stratum_defaults(morphology) + def _set_cluster_defaults(self, morph): + for system in morph.get('systems', {}): + if 'deploy-defaults' not in system: + system['deploy-defaults'] = {} + if 'deploy' not in system: + system['deploy'] = {} + def _set_system_defaults(self, morph): pass -- 1.8.5.rc2

On Mon, Nov 25, 2013 at 12:55:28PM +0000, Richard Maw wrote: In general, it's been my experience that it's better to not leave things like this (allowing empty systems) just in case they are needed in the future: right now, an empty system should be an error, and not something we silently accept from the user who made a mistake. If the future happens as we think it will, we should change the code then (and, in face, the validation then should be that the list of strata for the system and the list of strata for an initramfs can't both be empty, so this code would need to be changed anyway). Thus, I'd like this patch to be changed so it disallows an empty list of strata and a missing 'strata' field in a system morphology. I note that I was a silly person to use ValueError for that exception. Some way we should do something about it, but it's OK to leave it as is right now. -- http://www.codethink.co.uk/ http://wiki.baserock.org/ http://www.baserock.com/

It doesn't make sense to be able to specify an architecture from the IMPLEMENTS name, since you either need your architecture for something to build, or testarch for something that consistently doesn't build. --- yarns/architecture.yarn | 2 +- yarns/building.yarn | 4 ++-- yarns/implementations.yarn | 22 +++++++++++++++------- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/yarns/architecture.yarn b/yarns/architecture.yarn index 038492c..521575a 100644 --- a/yarns/architecture.yarn +++ b/yarns/architecture.yarn @@ -4,7 +4,7 @@ Morph Cross-Building Tests SCENARIO building a system for a different architecture GIVEN a workspace AND a git server - AND a system called base-system-testarch for architecture testarch in the git server + AND a system called base-system-testarch for the test architecture in the git server WHEN the user checks out the system branch called master AND the user attempts to build the system base-system-testarch in branch master THEN morph failed diff --git a/yarns/building.yarn b/yarns/building.yarn index b78c69c..6dad426 100644 --- a/yarns/building.yarn +++ b/yarns/building.yarn @@ -5,7 +5,7 @@ Morph Building Tests GIVEN a workspace AND a git server WHEN the user checks out the system branch called master - AND the user creates an uncommitted system morphology called base-system-testarch for architecture testarch in system branch master - AND the user attempts to build the system base-system-testarch in branch master + AND the user creates an uncommitted system morphology called base-system for our architecture in system branch master + AND the user attempts to build the system base-system in branch master THEN morph failed AND the build error message includes the string "Did you forget to commit it?" diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index 6491b38..cfe7b6a 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -127,11 +127,14 @@ another to hold a chunk. mkdir "$DATADIR/cache" mkdir "$DATADIR/tmp" - IMPLEMENTS GIVEN a system called (\S+) for architecture (\S+) in the git server +We need a consistent value for the architecture in some tests, so we +have a morphology using the test architecture. + + IMPLEMENTS GIVEN a system called (\S+) for the test architecture in the git server cat << EOF > "$DATADIR/gits/morphs/$MATCH_1.morph" - arch: $MATCH_2 + arch: testarch configuration-extensions: [] - description: A system called $MATCH_1 for architectures $MATCH_2 + description: A system called $MATCH_1 for test architecture kind: system name: $MATCH_1 strata: [] @@ -268,11 +271,16 @@ Editing morphologies with `morph edit`. cd "$DATADIR/workspace/$MATCH_3" attempt_morph edit "$MATCH_2" "$MATCH_1" - IMPLEMENTS WHEN the user creates an uncommitted system morphology called (\S+) for architecture (\S+) in system branch (\S+) - cat << EOF > "$DATADIR/workspace/$MATCH_3/test:morphs/$MATCH_1.morph" - arch: $MATCH_2 +To produce buildable morphologies, we need them to be of the same +architecture as the machine doing the testing. This uses `morph +print-architecture` to get a value appropriate for morph. + + IMPLEMENTS WHEN the user creates an uncommitted system morphology called (\S+) for our architecture in system branch (\S+) + arch=$(morph print-architecture) + cat << EOF > "$DATADIR/workspace/$MATCH_2/test:morphs/$MATCH_1.morph" + arch: $arch configuration-extensions: [] - description: A system called $MATCH_1 for architectures $MATCH_2 + description: A system called $MATCH_1 for architectures $arch kind: system name: $MATCH_1 strata: [] -- 1.8.5.rc2

The old build is still around for comparison. --- morphlib/plugins/branch_and_merge_plugin.py | 2 +- morphlib/plugins/build_plugin.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index 454caad..4666ea9 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -65,7 +65,7 @@ class BranchAndMergePlugin(cliapp.Plugin): self.app.add_subcommand('old-unpetrify', self.unpetrify) self.app.add_subcommand( 'tag', self.tag, arg_synopsis='TAG-NAME -- [GIT-COMMIT-ARG...]') - self.app.add_subcommand('build', self.build, + self.app.add_subcommand('old-build', self.build, arg_synopsis='SYSTEM') self.app.add_subcommand('old-status', self.status) self.app.add_subcommand('old-branch-from-image', diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py index 83226e5..8949d52 100644 --- a/morphlib/plugins/build_plugin.py +++ b/morphlib/plugins/build_plugin.py @@ -25,7 +25,7 @@ class BuildPlugin(cliapp.Plugin): def enable(self): self.app.add_subcommand('build-morphology', self.build_morphology, arg_synopsis='(REPO REF FILENAME)...') - self.app.add_subcommand('new-build', self.build, + self.app.add_subcommand('build', self.build, arg_synopsis='SYSTEM') def disable(self): -- 1.8.5.rc2

--- morphlib/plugins/branch_and_merge_plugin.py | 282 ---------------------------- 1 file changed, 282 deletions(-) diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index 4666ea9..f0ae656 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -65,8 +65,6 @@ class BranchAndMergePlugin(cliapp.Plugin): self.app.add_subcommand('old-unpetrify', self.unpetrify) self.app.add_subcommand( 'tag', self.tag, arg_synopsis='TAG-NAME -- [GIT-COMMIT-ARG...]') - self.app.add_subcommand('old-build', self.build, - arg_synopsis='SYSTEM') self.app.add_subcommand('old-status', self.status) self.app.add_subcommand('old-branch-from-image', self.branch_from_image, @@ -1528,286 +1526,6 @@ class BranchAndMergePlugin(cliapp.Plugin): self.reset_work_tree_safe(repo_dir) raise - def build(self, args): - '''Build a system image in the current system branch - - Command line arguments: - - * `SYSTEM` is the name of the system to build. - - This builds a system image, and any of its components that - need building. The system name is the basename of the system - morphology, in the root repository of the current system branch, - without the `.morph` suffix in the filename. - - The location of the resulting system image artifact is printed - at the end of the build output. - - You do not need to commit your changes before building, Morph - does that for you, in a temporary branch for each build. However, - note that Morph does not untracked files to the temporary branch, - only uncommitted changes to files git already knows about. You - need to `git add` and commit each new file yourself. - - Example: - - morph build devel-system-x86_64-generic - - ''' - - if len(args) != 1: - raise cliapp.AppException('morph build expects exactly one ' - 'parameter: the system to build') - - # Raise an exception if there is not enough space - morphlib.util.check_disk_available( - self.app.settings['tempdir'], - self.app.settings['tempdir-min-space'], - self.app.settings['cachedir'], - self.app.settings['cachedir-min-space']) - - system_name = args[0] - - # Deduce workspace and system branch and branch root repository. - workspace = self.deduce_workspace() - branch, branch_dir = self.deduce_system_branch() - branch_root = self.get_branch_config(branch_dir, 'branch.root') - branch_uuid = self.get_branch_config(branch_dir, 'branch.uuid') - - # Generate a UUID for the build. - build_uuid = uuid.uuid4().hex - - build_command = morphlib.buildcommand.BuildCommand(self.app) - build_command = self.app.hookmgr.call('new-build-command', - build_command) - push = self.app.settings['push-build-branches'] - - self.app.status(msg='Starting build %(uuid)s', uuid=build_uuid) - - self.app.status(msg='Collecting morphologies involved in ' - 'building %(system)s from %(branch)s', - system=system_name, branch=branch) - - # Get repositories of morphologies involved in building this system - # from the current system branch. - build_repos = self.get_system_build_repos( - branch, branch_dir, branch_root, system_name) - - # Generate temporary build ref names for all these repositories. - self.generate_build_ref_names(build_repos, branch_uuid) - - # Create the build refs for all these repositories and commit - # all uncommitted changes to them, updating all references - # to system branch refs to point to the build refs instead. - self.update_build_refs(build_repos, branch, build_uuid, push) - - if push: - self.push_build_refs(build_repos) - build_branch_root = branch_root - else: - dirname = build_repos[branch_root]['dirname'] - build_branch_root = urlparse.urljoin('file://', dirname) - - # Run the build. - build_command.build([build_branch_root, - build_repos[branch_root]['build-ref'], - system_name]) - - if push: - self.delete_remote_build_refs(build_repos) - - self.app.status(msg='Finished build %(uuid)s', uuid=build_uuid) - - def get_system_build_repos(self, system_branch, branch_dir, - branch_root, system_name): - '''Map upstream repository URLs to their checkouts in the system branch - - Also provides the list of morphologies stored in each repository, - grouped by kind. - - ''' - - build_repos = {} - - def prepare_repo_info(repo, dirname): - build_repos[repo] = { - 'dirname': dirname, - 'systems': [], - 'strata': [], - 'chunks': [] - } - - def add_morphology_info(info, category): - repo = info['repo'] or branch_root - if repo in build_repos: - repo_dir = build_repos[repo]['dirname'] - else: - repo_dir = self.find_repository(branch_dir, repo) - if repo_dir: - if not repo in build_repos: - prepare_repo_info(repo, repo_dir) - build_repos[repo][category].append(info['morph']) - return repo_dir - - # Add repository and morphology of the system. - branch_root_dir = self.find_repository(branch_dir, branch_root) - prepare_repo_info(branch_root, branch_root_dir) - build_repos[branch_root]['systems'].append(system_name) - - # Traverse and add repositories and morphologies involved in - # building this system from the system branch. - system_morphology = self.load_morphology(branch_root_dir, system_name) - for info in system_morphology['strata']: - if info['ref'] == system_branch or info['ref'] is None: - repo_dir = add_morphology_info(info, 'strata') - if repo_dir: - stratum_morphology = self.load_morphology( - repo_dir, info['morph']) - for info in stratum_morphology['chunks']: - if info['ref'] == system_branch: - add_morphology_info(info, 'chunks') - - return build_repos - - def inject_build_refs(self, morphology, build_repos, will_push): - # Starting from a system or stratum morphology, update all ref - # pointers of strata or chunks involved in a system build (represented - # by build_repos) to point to temporary build refs of the repos - # involved in the system build. - def inject_build_ref(info): - if info['repo'] in build_repos and ( - info['morph'] in build_repos[info['repo']]['strata'] or - info['morph'] in build_repos[info['repo']]['chunks']): - info['ref'] = build_repos[info['repo']]['build-ref'] - if not will_push: - dirname = build_repos[info['repo']]['dirname'] - info['repo'] = urlparse.urljoin('file://', dirname) - if morphology['kind'] == 'system': - for info in morphology['strata']: - inject_build_ref(info) - elif morphology['kind'] == 'stratum': - if morphology['build-depends'] is not None: - for info in morphology['build-depends']: - inject_build_ref(info) - for info in morphology['chunks']: - inject_build_ref(info) - - def generate_build_ref_names(self, build_repos, branch_uuid): - for repo, info in build_repos.iteritems(): - repo_dir = info['dirname'] - repo_uuid = self.get_repo_config(repo_dir, 'morph.uuid') - build_ref = os.path.join(self.app.settings['build-ref-prefix'], - branch_uuid, repo_uuid) - info['build-ref'] = build_ref - - def update_build_refs(self, build_repos, system_branch, build_uuid, - will_push): - '''Update build branches for each repository with any local changes ''' - - # Define the committer. - committer_name = 'Morph (on behalf of %s)' % \ - (morphlib.git.get_user_name(self.app.runcmd)) - committer_email = morphlib.git.get_user_email(self.app.runcmd) - - for repo, info in build_repos.iteritems(): - repo_dir = info['dirname'] - build_ref = info['build-ref'] - - self.app.status(msg='%(repo)s: Creating build branch', repo=repo) - - # Obtain parent SHA1 for the temporary ref tree to be committed. - # This will either be the current commit of the temporary ref or - # HEAD in case the temporary ref does not exist yet. - system_branch_sha1 = self.resolve_ref( - repo_dir, '%s^{commit}' % system_branch) - parent_sha1 = self.resolve_ref(repo_dir, '%s^{commit}' % build_ref) - if not parent_sha1: - parent_sha1 = system_branch_sha1 - - # Prepare an environment with our internal index file. - # This index file allows us to commit changes to a tree without - # git noticing any change in working tree or its own index. - env = dict(os.environ) - env['GIT_INDEX_FILE'] = os.path.join( - repo_dir, '.git', 'morph-index') - env['GIT_COMMITTER_NAME'] = committer_name - env['GIT_COMMITTER_EMAIL'] = committer_email - - # Read tree from current HEAD into the morph index. - self.app.runcmd(['git', 'read-tree', system_branch_sha1], - cwd=repo_dir, env=env) - - self.app.status(msg='%(repo)s: Adding uncommitted changes to ' - 'build branch', repo=repo) - - # Add all local, uncommitted changes to our internal index. - changed_files = self.get_uncommitted_changes(repo_dir, env) - self.app.runcmd(['git', 'add'] + changed_files, - cwd=repo_dir, env=env) - - self.app.status(msg='%(repo)s: Updating morphologies to use ' - 'build branch instead of "%(branch)s"', - repo=repo, branch=system_branch) - - # Update all references to the system branches of strata - # and chunks to point to the temporary refs, which is needed - # for building. - filenames = info['systems'] + info['strata'] - for filename in filenames: - # Inject temporary refs in the right places in each morphology. - morphology = self.load_morphology(repo_dir, filename) - self.inject_build_refs(morphology, build_repos, will_push) - with tempfile.NamedTemporaryFile(suffix='.morph') as f: - self.update_morphology( - repo_dir, filename, morphology, output_fd=f.file) - - morphology_sha1 = self.app.runcmd( - ['git', 'hash-object', '-t', 'blob', '-w', f.name], - cwd=repo_dir, env=env) - - try: - self.app.runcmd( - ['git', 'update-index', '--cacheinfo', - '100644', morphology_sha1, '%s.morph' % filename], - cwd=repo_dir, env=env) - except cliapp.AppException, e: - raise cliapp.AppException( - "You seem to want to build %s, but '%s.morph' " - "doesn't exist in the morphologies repository. " - "Did you forget to commit it?" % - (filename, filename)) - - # Create a commit message including the build UUID. This allows us - # to collect all commits of a build across repositories and thereby - # see the changes made to the entire system between any two builds. - message = 'Morph build %s\n\nSystem branch: %s\n' % \ - (build_uuid, system_branch) - - # Write and commit the tree and update the temporary build ref. - tree = self.app.runcmd( - ['git', 'write-tree'], cwd=repo_dir, env=env).strip() - commit = self.app.runcmd( - ['git', 'commit-tree', tree, '-p', parent_sha1], - feed_stdin=message, cwd=repo_dir, env=env).strip() - self.app.runcmd( - ['git', 'update-ref', '-m', message, - 'refs/heads/%s' % build_ref, commit], - cwd=repo_dir, env=env) - - def push_build_refs(self, build_repos): - for repo, info in build_repos.iteritems(): - self.app.status(msg='%(repo)s: Pushing build branch', repo=repo) - self.app.runcmd(['git', 'push', 'origin', '%s:%s' % - (info['build-ref'], info['build-ref'])], - cwd=info['dirname']) - - def delete_remote_build_refs(self, build_repos): - for repo, info in build_repos.iteritems(): - self.app.status(msg='%(repo)s: Deleting remote build branch', - repo=repo) - self.app.runcmd(['git', 'push', 'origin', - ':%s' % info['build-ref']], cwd=info['dirname']) - def status(self, args): '''Show information about the current system branch or workspace -- 1.8.5.rc2