3:28 p.m.
head: fa2282489f19507e3791437323e6f41bca2b6cde
branch: jonathanmaw/find-artifacts-rebase
repo: ssh://git@trove.baserock.org/baserock/baserock/morph
The previous patch submitted did not list everything required to rebuild
I.e. it only listed the artifact archive file, not any of the
accompanying files that contain metadata. This patch globs everything
that starts with the cache-key.
Since the last patch, the following changes were also made:
* The settings strings are more helpful.
* It only reads the files in the built image that end in .meta
* It uses self.output.write instead of print
Jonathan Maw (1):
Add find-artifacts script
scripts/find-artifacts | 123 +++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 123 insertions(+)
create mode 100755 scripts/find-artifacts
--
1.7.11.7
3:28 p.m.
New subject: [PATCH] Add find-artifacts script
This script reads a built baserock system and finds all the files in the
cache that are needed to replicate the build
---
scripts/find-artifacts | 123 +++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 123 insertions(+)
create mode 100755 scripts/find-artifacts
diff --git a/scripts/find-artifacts b/scripts/find-artifacts
new file mode 100755
index 0000000..a6f40a2
--- /dev/null
+++ b/scripts/find-artifacts
@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2011-2012 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# This is a program to read the metadata of a baserock image to produce a list
+# of all the chunks needed to build said image.
+
+import json
+import os
+import tempfile
+import gzip
+import sys
+import glob
+
+import cliapp
+
+path = os.path.dirname(os.path.dirname(__file__))
+sys.path.append(path)
+import morphlib
+
+# MountableImage yanked from the TrebuchetPlugin and modified slightly
+class MountableImage(object):
+ '''Mountable image (deals with decompression).
+
+ Note, this is a read-only mount in the sense that the decompressed
+ image is not then recompressed after, instead any changes are discarded.
+
+ '''
+ def __init__(self, app, artifact_path):
+ self.app = app
+ self.artifact_path = artifact_path
+
+ def setup(self, path):
+ (tempfd, self.temp_path) = \
+ tempfile.mkstemp(dir=self.app.settings['tempdir'])
+
+ try:
+ with os.fdopen(tempfd, "wb") as outfh:
+ infh = gzip.open(path, "rb")
+ morphlib.util.copyfileobj(infh, outfh)
+ infh.close()
+ except:
+ os.unlink(self.temp_path)
+ raise
+ part = morphlib.fsutils.setup_device_mapping(self.app.runcmd,
+ self.temp_path)
+ mount_point = tempfile.mkdtemp(dir=self.app.settings['tempdir'])
+ morphlib.fsutils.mount(self.app.runcmd, part, mount_point)
+ self.mount_point = mount_point
+ return mount_point
+
+ def cleanup(self, path, mount_point):
+ try:
+ morphlib.fsutils.unmount(self.app.runcmd, mount_point)
+ except:
+ pass
+ try:
+ morphlib.fsutils.undo_device_mapping(self.app.runcmd, path)
+ except:
+ pass
+ try:
+ os.rmdir(mount_point)
+ os.unlink(path)
+ except:
+ pass
+
+ def __enter__(self):
+ return self.setup(self.artifact_path)
+
+ def __exit__(self, exctype, excvalue, exctraceback):
+ self.cleanup(self.temp_path, self.mount_point)
+
+
+class FindArtifacts(cliapp.Application):
+
+ def add_settings(self):
+ self.settings.string(['cachedir'], 'The directory morph writes its
'
+ 'cache to')
+ self.settings.string(['tempdir'], 'A temporary directory to mount the
'
+ 'system image in')
+
+ def process_args(self, args):
+ artifacts_dir = os.path.join(self.settings['cachedir'],
'artifacts')
+
+ # args[0] is the path to the built image.
+ # Mount the image
+ mount_point = None
+ with MountableImage(self, args[0]) as mount_point:
+ # For each meta file:
+ metadir = os.path.join(mount_point, 'factory-run',
'baserock')
+ metaglob = os.path.join(metadir, '*.meta')
+ for metafile in glob.glob(metaglob):
+ metafilepath = os.path.join(metadir, metafile)
+ # Read the file as JSON and extract the kind and cache-key
+ metajson = json.load(open(metafilepath))
+ cache_key = metajson['cache-key']
+
+ # Grab every file in the artifact cache which matches the
+ # cache-key
+ found_artifact = False
+ artifact_glob = os.path.join(artifacts_dir, cache_key) + '*'
+ for cached_file in glob.glob(artifact_glob):
+ found_artifact = True
+ self.output.write(cached_file + "\n")
+ if found_artifact == False:
+ raise cliapp.AppException('Could not find cache-key '
+ + cache_key + ' for artifact '
+ + metajson['artifact-name'])
+
+FindArtifacts().run()
--
1.7.11.7
3:35 p.m.
New subject: [PATCH] Add find-artifacts script
On Fri, Nov 30, 2012 at 02:28:24PM +0000, Jonathan Maw wrote:
This script reads a built baserock system and finds all the files in
the
cache that are needed to replicate the build
---
scripts/find-artifacts | 123 +++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 123 insertions(+)
create mode 100755 scripts/find-artifacts
diff --git a/scripts/find-artifacts b/scripts/find-artifacts
new file mode 100755
index 0000000..a6f40a2
--- /dev/null
+++ b/scripts/find-artifacts
@@ -0,0 +1,123 @@
snip
+ def process_args(self, args):
+ artifacts_dir = os.path.join(self.settings['cachedir'],
'artifacts')
+
+ # args[0] is the path to the built image.
+ # Mount the image
+ mount_point = None
+ with MountableImage(self, args[0]) as mount_point:
+ # For each meta file:
+ metadir = os.path.join(mount_point, 'factory-run',
'baserock')
+ metaglob = os.path.join(metadir, '*.meta')
+ for metafile in glob.glob(metaglob):
glob.iglob is a bit better here, it returns an iterator rather than a
list. It uses less memory, but otherwise performs the same when used
here.
Not that it matters too much here, since it won't be a vast amount of
memory and memory isn't a scarce resource here, but it's a useful thing
to remember.
+ metafilepath = os.path.join(metadir, metafile)
+ # Read the file as JSON and extract the kind and cache-key
+ metajson = json.load(open(metafilepath))
+ cache_key = metajson['cache-key']
This leaves the file open until garbage collected, since the
opened file isn't referenced anyehere when json.load() exits.
This could potentially lead to a crash if there are a lot of metafiles
and you end up opening more files than your process is allowed to,
though I'll admit that it's unlikely to process that many files without
garbage collecting.
CPython's garbage collection is reference counted, so it's going to be
closed as soon as json.load() exits, but other python implementations
may garbage collect without reference counting, so it can happen at any
time after it is no longer referenced.
The point of this rant, is that in some rare occasions this will cause
the program to crash, but really I just want you to use the following
because I think it looks nicer, is guaranteed to close the file after
the with block, and it guarantees to close the file when you have an
exception.
with open(metafilepath) as metafile:
metajson = json.load(metafile)
+
+ # Grab every file in the artifact cache which matches the
+ # cache-key
+ found_artifact = False
+ artifact_glob = os.path.join(artifacts_dir, cache_key) + '*'
+ for cached_file in glob.glob(artifact_glob):
+ found_artifact = True
+ self.output.write(cached_file + "\n")
+ if found_artifact == False:
+ raise cliapp.AppException('Could not find cache-key '
+ + cache_key + ' for artifact '
+ + metajson['artifact-name'])
You could use iglob here too, but it may be nicer to eliminate the
found_artifact variable.
found_artifacts = glob.glob(artifact_glob)
if not found_artifacts:
raise cliapp.AppException(...)
for cached_file in found_artifacts:
self.output.write(...)
I don't have a preference for either approach, but the latter is one
less line and one less variable.
3755
days inactive
3762
days old
2 comments
2 participants
participants (2)
-
Jonathan Maw -
Richard Maw