This provides access to the /baserock directory as if it were a dict, abstracting away the details of how to get data out of it. The abstraction is useful since it is easier to use than accessing /baserock yourself, and allows the storage format to be changed more easily. --- morphlib/__init__.py | 1 + morphlib/systemmetadatadir.py | 63 +++++++++++++++++++++++++++++++++ morphlib/systemmetadatadir_tests.py | 66 +++++++++++++++++++++++++++++++++++ 3 files changed, 130 insertions(+) create mode 100644 morphlib/systemmetadatadir.py create mode 100644 morphlib/systemmetadatadir_tests.py diff --git a/morphlib/__init__.py b/morphlib/__init__.py index b1e3c7c..7eb3f97 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -78,6 +78,7 @@ import sourcepool import stagingarea import stopwatch import sysbranchdir +import systemmetadatadir import tempdir import util import workspace diff --git a/morphlib/systemmetadatadir.py b/morphlib/systemmetadatadir.py new file mode 100644 index 0000000..17ca839 --- /dev/null +++ b/morphlib/systemmetadatadir.py @@ -0,0 +1,63 @@ +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + + +import collections +import glob +import json +import os + + +class SystemMetadataDir(collections.MutableMapping): + + '''An abstraction over the /baserock metadata directory. + + This allows methods of iterating over it, and accessing it like + a dict. + + ''' + + def __init__(self, metadata_path): + collections.MutableMapping.__init__(self) + self._metadata_path = metadata_path + + def _join_path(self, *args): + return os.path.join(self._metadata_path, *args) + + def _raw_path_iter(self): + return glob.iglob(self._join_path('*.meta')) + + def __getitem__(self, key): + try: + with open(self._join_path('%s.meta' % key), 'r') as f: + return json.load(f) + except IOError: + raise KeyError(key) + + def __setitem__(self, key, value): + with open(self._join_path('%s.meta' % key), 'w') as f: + json.dump(value, f, indent=4, sort_keys=True) + + def __delitem__(self, key): + os.unlink(self._join_path('%s.meta' % key)) + + def __iter__(self): + return (os.path.basename(fn)[:-len('.meta')] + for fn in self._raw_path_iter()) + + def __len__(self): + return len(list(self._raw_path_iter())) diff --git a/morphlib/systemmetadatadir_tests.py b/morphlib/systemmetadatadir_tests.py new file mode 100644 index 0000000..1cb1ed3 --- /dev/null +++ b/morphlib/systemmetadatadir_tests.py @@ -0,0 +1,66 @@ +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + + +import os +import shutil +import tempfile +import unittest + +import morphlib + + +class SystemMetadataDirTests(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.metadatadir = os.path.join(self.tempdir, 'baserock') + os.mkdir(self.metadatadir) + self.smd = morphlib.systemmetadatadir.SystemMetadataDir(self.metadatadir) + + def tearDown(self): + shutil.rmtree(self.tempdir) + + def test_add_new(self): + self.smd['key'] = {'foo': 'bar'} + self.assertEqual(self.smd['key']['foo'], 'bar') + + def test_replace(self): + self.smd['key'] = {'foo': 'bar'} + self.smd['key'] = {'foo': 'baz'} + self.assertEqual(self.smd['key']['foo'], 'baz') + + def test_remove(self): + self.smd['key'] = {'foo': 'bar'} + del self.smd['key'] + self.assertTrue('key' not in self.smd) + + def test_iterate(self): + self.smd['build-essential'] = "Some data" + self.smd['core'] = "More data" + self.smd['foundation'] = "Yet more data" + self.assertEqual(sorted(self.smd.keys()), + ['build-essential', 'core', 'foundation']) + self.assertEqual(dict(self.smd.iteritems()), + { + 'build-essential': "Some data", + 'core': "More data", + 'foundation': "Yet more data", + }) + + def test_raises_KeyError(self): + self.assertRaises(KeyError, lambda: self.smd['key']) -- 1.7.10.4

This changes the interface of branch-from-image to only take 1 parameter, the name of the new system branch, as the root repository is loaded from the metadata. This was also what the previous version of branch-from-image did, but that silently ignored the parameter. Given there are not many users of branch-from-image, I felt it was a reasonable change. --- morphlib/plugins/branch_and_merge_new_plugin.py | 98 +++++++++++++++++++++++ morphlib/plugins/branch_and_merge_plugin.py | 12 +-- tests.as-root/branch-from-image-works.script | 3 +- 3 files changed, 102 insertions(+), 11 deletions(-) diff --git a/morphlib/plugins/branch_and_merge_new_plugin.py b/morphlib/plugins/branch_and_merge_new_plugin.py index 0e01111..12ce0fe 100644 --- a/morphlib/plugins/branch_and_merge_new_plugin.py +++ b/morphlib/plugins/branch_and_merge_new_plugin.py @@ -56,6 +56,15 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): arg_synopsis='-- COMMAND [ARGS...]') self.app.add_subcommand('status', self.status, arg_synopsis='') + self.app.add_subcommand('branch-from-image', self.branch_from_image, + arg_synopsis='REPO BRANCH') + group_branch = 'Branching Options' + self.app.settings.string(['metadata-dir'], + 'Set metadata location for branch-from-image' + ' (default: /baserock)', + metavar='DIR', + default='/baserock', + group=group_branch) def disable(self): pass @@ -788,3 +797,92 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): if not has_uncommitted_changes: self.app.output.write("\nNo repos have outstanding changes.\n") + + def branch_from_image(self, args): + '''Produce a branch of an existing system image. + + Given the metadata + + ''' + if len(args) != 1: + raise cliapp.AppException( + 'branch-from-image needs a branch name') + system_branch = args[0] + metadata_path = self.app.settings['metadata-dir'] + alias_resolver = morphlib.repoaliasresolver.RepoAliasResolver( + self.app.settings['repo-alias']) + + self._require_git_user_config() + + ws = morphlib.workspace.open('.') + + system, metadata = self._load_system_metadata(metadata_path) + resolved_refs = dict(self._resolve_refs_from_metadata(alias_resolver, + metadata)) + logging.debug('Resolved refs: %r' % resolved_refs) + base_ref = system['sha1'] + # The previous version would fall back to deducing this from the repo + # url and the repo alias resolver, but this does not always work, and + # new systems always have repo-alias in the metadata + root_url = system['repo-alias'] + + lrc, rrc = morphlib.util.new_repo_caches(self.app) + cached_repo = lrc.get_updated_repo(root_url) + + + with self._initializing_system_branch( + ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd): + + # TODO: It's nasty to clone to a sha1 then create a branch + # of that sha1 then check it out, a nicer API may be the + # initial clone not checking out a branch at all, then + # the user creates and checks out their own branches + gd.branch(system_branch, base_ref) + gd.checkout(system_branch) + + loader = morphlib.morphloader.MorphologyLoader() + morphs = self._load_all_sysbranch_morphologies(sb, loader) + + morphs.repoint_refs(sb.root_repository_url, + sb.system_branch_name) + + morphs.petrify_chunks(resolved_refs) + + self._save_dirty_morphologies(loader, sb, morphs.morphologies) + + @staticmethod + def _load_system_metadata(path): + '''Load all metadata in `path` corresponding to a single System. + ''' + + smd = morphlib.systemmetadatadir.SystemMetadataDir(path) + metadata = smd.values() + systems = [md for md in metadata + if 'kind' in md and md['kind'] == 'system'] + + if not systems: + raise cliapp.AppException( + 'Metadata directory does not contain any systems.') + if len(systems) > 1: + raise cliapp.AppException( + 'Metadata directory contains multiple systems.') + system_metadatum = systems[0] + + metadata_cache_id_lookup = dict((md['cache-key'], md) + for md in metadata) + + return system_metadatum, metadata_cache_id_lookup + + @staticmethod + def _resolve_refs_from_metadata(alias_resolver, metadata): + '''Pre-resolve a set of refs from existing metadata. + + Given the metadata, generate a mapping of all the (repo, ref) + pairs defined in the metadata and the commit id they resolved to. + + ''' + for md in metadata.itervalues(): + repourls = set((md['repo-alias'], md['repo'])) + repourls.update(alias_resolver.aliases_from_url(md['repo'])) + for repourl in repourls: + yield ((repourl, md['original_ref']), md['sha1']) diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index 37d5e40..2f8560d 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -68,15 +68,9 @@ class BranchAndMergePlugin(cliapp.Plugin): self.app.add_subcommand('build', self.build, arg_synopsis='SYSTEM') self.app.add_subcommand('old-status', self.status) - self.app.add_subcommand('branch-from-image', self.branch_from_image, - arg_synopsis='REPO BRANCH') - group_branch = 'Branching Options' - self.app.settings.string(['metadata-dir'], - 'Set metadata location for branch-from-image' - ' (default: /baserock)', - metavar='DIR', - default='/baserock', - group=group_branch) + self.app.add_subcommand('old-branch-from-image', + self.branch_from_image, + arg_synopsis='REPO BRANCH') # Advanced commands self.app.add_subcommand('old-foreach', self.foreach, diff --git a/tests.as-root/branch-from-image-works.script b/tests.as-root/branch-from-image-works.script index 9f82f62..942301e 100755 --- a/tests.as-root/branch-from-image-works.script +++ b/tests.as-root/branch-from-image-works.script @@ -48,8 +48,7 @@ git commit --quiet -m 'Make hello say goodbye' workspace="$DATADIR/workspace" "$SRCDIR/scripts/test-morph" init "$workspace" cd "$workspace" -"$SRCDIR/scripts/test-morph" branch-from-image \ - test:morphs mybranch \ +"$SRCDIR/scripts/test-morph" branch-from-image mybranch \ --metadata-dir="$extracted/baserock" cd mybranch/test:morphs grep -qFe "$hello_chunk_commit" hello-stratum.morph -- 1.7.10.4