8:51 a.m.
Repo: git://git.baserock.org/baserock/baserock/morph.git
Ref: baserock/richardmaw/S7905-morph-gc
Sha1: fac3e935a4b7eee89bf525adeeb90aafe069e626
Land: master
This adds methods to the local artifact cache to inspect its contents
and delete artifacts by cache key.
Policy for deletion is left to the gc plugin.
Staging areas of failed builds are removed first from --tempdir, and
then removes all extracted chunks if there is still not enough space.
Artifacts are removed from the cache, oldest first, until there are
no longer any chunks older than --cachedir-artifact-max-age.
It will finish deleting when there is enough space.
I have tested this with a tmpfs, a script to generate random artifacts,
and touch. A proper black-box test would be difficult to implement, and
would require root, or user namespace support.
Richard Maw (5):
util: split out space unify logic
lac_tests: set cache key
LAC: implement contents listing and removal by key
lac_tests: Cover list and delete
plugins: `morph gc` command via plugin
morphlib/localartifactcache.py | 28 ++++++++
morphlib/localartifactcache_tests.py | 36 +++++++++-
morphlib/plugins/gc_plugin.py | 121 ++++++++++++++++++++++++++++++++++
morphlib/util.py | 25 +++++--
without-test-modules | 1 +
5 files changed, 204 insertions(+), 7 deletions(-)
create mode 100644 morphlib/plugins/gc_plugin.py
--
1.7.10.4
8:51 a.m.
New subject: [PATCH 1/5] util: split out space unify logic
---
morphlib/util.py | 25 +++++++++++++++++++------
1 file changed, 19 insertions(+), 6 deletions(-)
diff --git a/morphlib/util.py b/morphlib/util.py
index 1a347da..6dcb22b 100644
--- a/morphlib/util.py
+++ b/morphlib/util.py
@@ -237,16 +237,29 @@ def on_same_filesystem(path_a, path_b): # pragma: no cover
# TODO: return true if one path is a subvolume of the other on btrfs?
return os.stat(path_a).st_dev == os.stat(path_b).st_dev
+def unify_space_requirements(tmp_path, tmp_min_size,
+ cache_path, cache_min_size): # pragma: no cover
+ """Adjust minimum sizes when paths share a disk.
+
+ Given pairs of path and minimum size, return the minimum sizes such
+ that when the paths are on the same disk, the sizes are added together.
+
+ """
+ # TODO: make this work for variable number of (path, size) pairs as needed
+ # hint: try list.sort and itertools.groupby
+ if not on_same_filesystem(tmp_path, cache_path):
+ return tmp_min_size, cache_min_size
+ unified_size = tmp_min_size + cache_min_size
+ return unified_size, unified_size
+
def check_disk_available(tmp_path, tmp_min_size,
cache_path, cache_min_size): # pragma: no cover
# if both are on the same filesystem, assume they share a storage pool,
# so the sum of the two sizes needs to be available
- # TODO: if we need to do this on any more than 2 filesystems
- # extend it to take a [(path, min)] and do some arcane mathematics
- # to split it into groups that share a filesystem
- # hint: try list.sort and itertools.groupby
- if on_same_filesystem(tmp_path, cache_path):
- tmp_min_size = cache_min_size = tmp_min_size + cache_min_size
+ # TODO: if we need to do this on any more than 2 paths
+ # extend it to take a [(path, min)]
+ tmp_min_size, cache_min_size = unify_space_requirements(
+ tmp_path, tmp_min_size, cache_path, cache_min_size)
tmp_size, cache_size = map(get_bytes_free_in_path, (tmp_path, cache_path))
errors = []
for path, min in [(tmp_path, tmp_min_size), (cache_path, cache_min_size)]:
--
1.7.10.4
8:51 a.m.
New subject: [PATCH 2/5] lac_tests: set cache key
---
morphlib/localartifactcache_tests.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/morphlib/localartifactcache_tests.py b/morphlib/localartifactcache_tests.py
index 36b5e89..10f3ab6 100644
--- a/morphlib/localartifactcache_tests.py
+++ b/morphlib/localartifactcache_tests.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012 Codethink Limited
+# Copyright (C) 2012,2013 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -47,8 +47,10 @@ class LocalArtifactCacheTests(unittest.TestCase):
'repo', 'ref', 'sha1', 'tree', morph,
'chunk.morph')
self.runtime_artifact = morphlib.artifact.Artifact(
self.source, 'chunk-runtime')
+ self.runtime_artifact.cache_key = '0'*64
self.devel_artifact = morphlib.artifact.Artifact(
self.source, 'chunk-devel')
+ self.devel_artifact.cache_key = '0'*64
def tearDown(self):
self.tempdir.remove()
--
1.7.10.4
8:51 a.m.
New subject: [PATCH 3/5] LAC: implement contents listing and removal by key
---
morphlib/localartifactcache.py | 28 ++++++++++++++++++++++++++++
1 file changed, 28 insertions(+)
diff --git a/morphlib/localartifactcache.py b/morphlib/localartifactcache.py
index b845ceb..d8abf2a 100644
--- a/morphlib/localartifactcache.py
+++ b/morphlib/localartifactcache.py
@@ -14,6 +14,7 @@
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+import collections
import os
import morphlib
@@ -115,3 +116,30 @@ class LocalArtifactCache(object):
for basename in basenames:
os.remove(os.path.join(dirname, basename))
+ def list_contents(self): # pragma: no cover
+ '''Return the set of sources cached and related information.
+
+ returns a [(cache_key, set(artifacts), last_used)]
+
+ '''
+ CacheInfo = collections.namedtuple('CacheInfo', ('artifacts',
'mtime'))
+ contents = collections.defaultdict(lambda: CacheInfo(set(), 0))
+ for dirpath, dirnames, filenames in os.walk(self.cachedir):
+ for filename in filenames:
+ cachekey = filename[:63]
+ artifact = filename[65:]
+ artifacts, max_mtime = contents[cachekey]
+ artifacts.add(artifact)
+ this_mtime = os.stat(os.path.join(dirpath, filename)).st_mtime
+ contents[cachekey] = CacheInfo(artifacts,
+ max(max_mtime, this_mtime))
+ return ((cache_key, info.artifacts, info.mtime)
+ for cache_key, info in contents.iteritems())
+
+
+ def remove(self, cachekey): # pragma: no cover
+ '''Remove all artifacts associated with the given
cachekey.'''
+ for dirpath, dirnames, filenames in os.walk(self.cachedir):
+ for filename in filenames:
+ if filename.startswith(cachekey):
+ os.remove(os.path.join(dirpath, filename))
--
1.7.10.4
8:51 a.m.
New subject: [PATCH 4/5] lac_tests: Cover list and delete
---
morphlib/localartifactcache.py | 4 ++--
morphlib/localartifactcache_tests.py | 32 ++++++++++++++++++++++++++++++++
2 files changed, 34 insertions(+), 2 deletions(-)
diff --git a/morphlib/localartifactcache.py b/morphlib/localartifactcache.py
index d8abf2a..76d085d 100644
--- a/morphlib/localartifactcache.py
+++ b/morphlib/localartifactcache.py
@@ -116,7 +116,7 @@ class LocalArtifactCache(object):
for basename in basenames:
os.remove(os.path.join(dirname, basename))
- def list_contents(self): # pragma: no cover
+ def list_contents(self):
'''Return the set of sources cached and related information.
returns a [(cache_key, set(artifacts), last_used)]
@@ -137,7 +137,7 @@ class LocalArtifactCache(object):
for cache_key, info in contents.iteritems())
- def remove(self, cachekey): # pragma: no cover
+ def remove(self, cachekey):
'''Remove all artifacts associated with the given
cachekey.'''
for dirpath, dirnames, filenames in os.walk(self.cachedir):
for filename in filenames:
diff --git a/morphlib/localartifactcache_tests.py b/morphlib/localartifactcache_tests.py
index 10f3ab6..082b926 100644
--- a/morphlib/localartifactcache_tests.py
+++ b/morphlib/localartifactcache_tests.py
@@ -157,3 +157,35 @@ class LocalArtifactCacheTests(unittest.TestCase):
cache.clear()
self.assertFalse(cache.has(self.runtime_artifact))
+ def test_put_artifacts_and_list_them_afterwards(self):
+ cache = morphlib.localartifactcache.LocalArtifactCache(
+ self.tempdir.dirname)
+
+ handle = cache.put(self.runtime_artifact)
+ handle.write('runtime')
+ handle.close()
+
+ self.assertTrue(len(list(cache.list_contents())) == 1)
+
+ handle = cache.put(self.devel_artifact)
+ handle.write('devel')
+ handle.close()
+
+ self.assertTrue(len(list(cache.list_contents())) == 1)
+
+ def test_put_artifacts_and_remove_them_afterwards(self):
+ cache = morphlib.localartifactcache.LocalArtifactCache(
+ self.tempdir.dirname)
+
+ handle = cache.put(self.runtime_artifact)
+ handle.write('runtime')
+ handle.close()
+
+ handle = cache.put(self.devel_artifact)
+ handle.write('devel')
+ handle.close()
+
+ key = list(cache.list_contents())[0][0]
+ cache.remove(key)
+
+ self.assertTrue(len(list(cache.list_contents())) == 0)
--
1.7.10.4
8:51 a.m.
New subject: [PATCH 5/5] plugins: `morph gc` command via plugin
This removes staging areas and extracted chunks from --tempdir.
Then asks the local artifact cache what artifacts it has and how
old they are, removing sources older than the date specified,
oldest first, until there is sufficient space.
---
morphlib/plugins/gc_plugin.py | 121 +++++++++++++++++++++++++++++++++++++++++
without-test-modules | 1 +
2 files changed, 122 insertions(+)
create mode 100644 morphlib/plugins/gc_plugin.py
diff --git a/morphlib/plugins/gc_plugin.py b/morphlib/plugins/gc_plugin.py
new file mode 100644
index 0000000..065877a
--- /dev/null
+++ b/morphlib/plugins/gc_plugin.py
@@ -0,0 +1,121 @@
+# Copyright (C) 2013 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import logging
+import os
+import shutil
+import time
+
+import cliapp
+
+import morphlib
+
+
+class GCPlugin(cliapp.Plugin):
+
+ def enable(self):
+ self.app.add_subcommand('gc', self.gc,
+ arg_synopsis='')
+ self.app.settings.integer(['cachedir-artifact-max-age'],
+ 'how long to keep artifacts around for '
+ 'in seconds, (default: 1 week)',
+ metavar='PERIOD',
+ group="Storage Options",
+ default=(60*60*24*7))
+
+ def disable(self):
+ pass
+
+ def gc(self, args):
+ '''Make space by removing unused files.
+
+ This removes artifacts that have not been used recently,
+ where recently is defined by --cachedir-artifact-max-age.
+
+ This removes extracted chunks and staging areas for failed builds
+ from the directory specified by --tempdir.
+
+ '''
+
+ tempdir = self.app.settings['tempdir']
+ cachedir = self.app.settings['cachedir']
+ tempdir_min_space, cachedir_min_space = \
+ morphlib.util.unify_space_requirements(
+ tempdir, self.app.settings['tempdir-min-space'],
+ cachedir, self.app.settings['cachedir-min-space'])
+
+ self.cleanup_tempdir(tempdir, tempdir_min_space)
+ self.cleanup_cachedir(cachedir, cachedir_min_space)
+
+ def cleanup_tempdir(self, temp_path, min_space):
+ self.app.status(msg='Cleaning up temp dir %(temp_path)s',
+ temp_path=temp_path, chatty=True)
+ for subdir in ('failed', 'chunks'):
+ if morphlib.util.get_bytes_free_in_path(temp_path) >= min_space:
+ self.app.status(msg='Not Removing subdirectory '
+ '%(subdir)s, enough space already cleared',
+ subdir=os.path.join(temp_path, subdir),
+ chatty=True)
+ break
+ self.app.status('Removing temp subdirectory: %(subdir)s',
+ subdir=subdir)
+ path = os.path.join(temp_path, subdir)
+ if os.path.exists(path):
+ shutil.rmtree(path)
+
+ def calculate_latest_deletable_date(self):
+ return time.time() - self.app.settings['cachedir-artifact-max-age']
+
+ def find_deletable_artifacts(self, lac, timestamp):
+ '''Get a list of cache keys in order of how old they
are.'''
+ contents = lac.list_contents()
+ deletable = ((cachekey, mtime)
+ for cachekey, artifacts, mtime in contents
+ if mtime < timestamp)
+ return [cachekey for cachekey, mtime in sorted(deletable, key=lambda x: x[1])]
+
+ def cleanup_cachedir(self, cache_path, min_space):
+ if morphlib.util.get_bytes_free_in_path(cache_path) >= min_space:
+ self.app.status(msg='Not cleaning up cachedir, '
+ 'sufficient space already cleared',
+ chatty=True)
+ return
+ lac = morphlib.localartifactcache.LocalArtifactCache(cache_path)
+ latest_deletable = self.calculate_latest_deletable_date()
+ logging.debug('Can remove artifacts older than timestamp %d'
+ % latest_deletable)
+ deletable = self.find_deletable_artifacts(lac, latest_deletable)
+ logging.debug('Can remove artifacts %s' % repr(deletable))
+ for i, cachekey in enumerate(deletable, start=1):
+ self.app.status(msg='Removing source %(cachekey)s',
+ cachekey=cachekey, chatty=True)
+ lac.remove(cachekey)
+ if morphlib.util.get_bytes_free_in_path(cache_path) >= min_space:
+ self.app.status(msg='Finished cleaning up cachedir with '
+ '%(remaining)d old sources remaining',
+ remaining=(len(deletable) - i), chatty=True)
+ break
+ if morphlib.util.get_bytes_free_in_path(cache_path) >= min_space:
+ self.app.status(msg='Made sufficient space in %(cache_path)s '
+ 'after removing %(sources)d sources',
+ sources=len(deletable), cache_path=cache_path)
+ self.app.status(msg='Unable to clear enough space in %(cache_path)s '
+ 'after removing %(sources)d sources. '
+ 'Please reduce cachedir-artifact-max-age, '
+ 'clear space from elsewhere, enlarge the disk '
+ 'or reduce cachedir-min-space.',
+ cache_path=cache_path, sources=len(deletable),
+ error=True)
diff --git a/without-test-modules b/without-test-modules
index 89b0bd8..f143eb4 100644
--- a/without-test-modules
+++ b/without-test-modules
@@ -26,3 +26,4 @@ morphlib/plugins/__init__.py
morphlib/writeexts.py
morphlib/plugins/copy-artifacts_plugin.py
morphlib/plugins/trovectl_plugin.py
+morphlib/plugins/gc_plugin.py
--
1.7.10.4
8:57 a.m.
On 07/06/2013 13:51, Richard Maw wrote:
Repo: git://git.baserock.org/baserock/baserock/morph.git
Ref: baserock/richardmaw/S7905-morph-gc
Sha1: fac3e935a4b7eee89bf525adeeb90aafe069e626
Land: master
This adds methods to the local artifact cache to inspect its contents
and delete artifacts by cache key.
Policy for deletion is left to the gc plugin.
does the user have to *run* this? can it get run by morph, or by the
system somehow?
9:07 a.m.
On Fri, Jun 07, 2013 at 01:57:25PM +0100, Paul Sherwood wrote:
On 07/06/2013 13:51, Richard Maw wrote:
> Repo: git://git.baserock.org/baserock/baserock/morph.git
> Ref: baserock/richardmaw/S7905-morph-gc
> Sha1: fac3e935a4b7eee89bf525adeeb90aafe069e626
> Land: master
>
> This adds methods to the local artifact cache to inspect its contents
> and delete artifacts by cache key.
> Policy for deletion is left to the gc plugin.
does the user have to *run* this? can it get run by morph, or by the
system somehow?
The user does have to run this themselves. Given how new it is, I'm not
confident we should make it automatic yet.
People could add it to crontab, or a systemd timer unit, but right now
I'm happy with people running it when morph tells them they're low on
disk space.
9:18 a.m.
On Fri, Jun 07, 2013 at 02:16:20PM +0100, Paul Sherwood wrote:
On 07/06/2013 14:07, Richard Maw wrote:
> I'm happy with people running it when morph tells them they're low on
> disk space.
fair enough - provided the out-of-disk space error message tells them to
do so :-)
Good point, while it says to clean up some space, it doesn't currently say
that you can use `morph gc`. I'll prepare a second patch series.
I'd appreciate feedback for the names of the configuration items,
I think they're awful, but couldn't think of anything better.
10 a.m.
New subject: [PATCH 0/6] Add morph gc subcommand
Repo: git://git.baserock.org/baserock/baserock/morph.git
Ref: baserock/richardmaw/S7905-morph-gc
Sha1: 7cff6ae3566f38894f5d9a3c74d014db6e86004d
Land: master
Richard Maw (6):
util: split out space unify logic
lac_tests: set cache key
LAC: implement contents listing and removal by key
lac_tests: Cover list and delete
plugins: `morph gc` command via plugin
util: suggest how to deal with insufficient space
morphlib/localartifactcache.py | 28 ++++++++
morphlib/localartifactcache_tests.py | 36 +++++++++-
morphlib/plugins/gc_plugin.py | 122 ++++++++++++++++++++++++++++++++++
morphlib/util.py | 33 +++++++--
without-test-modules | 1 +
5 files changed, 212 insertions(+), 8 deletions(-)
create mode 100644 morphlib/plugins/gc_plugin.py
--
1.7.10.4
10 a.m.
New subject: [PATCH 1/6] util: split out space unify logic
---
morphlib/util.py | 25 +++++++++++++++++++------
1 file changed, 19 insertions(+), 6 deletions(-)
diff --git a/morphlib/util.py b/morphlib/util.py
index 1a347da..6dcb22b 100644
--- a/morphlib/util.py
+++ b/morphlib/util.py
@@ -237,16 +237,29 @@ def on_same_filesystem(path_a, path_b): # pragma: no cover
# TODO: return true if one path is a subvolume of the other on btrfs?
return os.stat(path_a).st_dev == os.stat(path_b).st_dev
+def unify_space_requirements(tmp_path, tmp_min_size,
+ cache_path, cache_min_size): # pragma: no cover
+ """Adjust minimum sizes when paths share a disk.
+
+ Given pairs of path and minimum size, return the minimum sizes such
+ that when the paths are on the same disk, the sizes are added together.
+
+ """
+ # TODO: make this work for variable number of (path, size) pairs as needed
+ # hint: try list.sort and itertools.groupby
+ if not on_same_filesystem(tmp_path, cache_path):
+ return tmp_min_size, cache_min_size
+ unified_size = tmp_min_size + cache_min_size
+ return unified_size, unified_size
+
def check_disk_available(tmp_path, tmp_min_size,
cache_path, cache_min_size): # pragma: no cover
# if both are on the same filesystem, assume they share a storage pool,
# so the sum of the two sizes needs to be available
- # TODO: if we need to do this on any more than 2 filesystems
- # extend it to take a [(path, min)] and do some arcane mathematics
- # to split it into groups that share a filesystem
- # hint: try list.sort and itertools.groupby
- if on_same_filesystem(tmp_path, cache_path):
- tmp_min_size = cache_min_size = tmp_min_size + cache_min_size
+ # TODO: if we need to do this on any more than 2 paths
+ # extend it to take a [(path, min)]
+ tmp_min_size, cache_min_size = unify_space_requirements(
+ tmp_path, tmp_min_size, cache_path, cache_min_size)
tmp_size, cache_size = map(get_bytes_free_in_path, (tmp_path, cache_path))
errors = []
for path, min in [(tmp_path, tmp_min_size), (cache_path, cache_min_size)]:
--
1.7.10.4
10 a.m.
New subject: [PATCH 2/6] lac_tests: set cache key
---
morphlib/localartifactcache_tests.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/morphlib/localartifactcache_tests.py b/morphlib/localartifactcache_tests.py
index 36b5e89..10f3ab6 100644
--- a/morphlib/localartifactcache_tests.py
+++ b/morphlib/localartifactcache_tests.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012 Codethink Limited
+# Copyright (C) 2012,2013 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -47,8 +47,10 @@ class LocalArtifactCacheTests(unittest.TestCase):
'repo', 'ref', 'sha1', 'tree', morph,
'chunk.morph')
self.runtime_artifact = morphlib.artifact.Artifact(
self.source, 'chunk-runtime')
+ self.runtime_artifact.cache_key = '0'*64
self.devel_artifact = morphlib.artifact.Artifact(
self.source, 'chunk-devel')
+ self.devel_artifact.cache_key = '0'*64
def tearDown(self):
self.tempdir.remove()
--
1.7.10.4
10 a.m.
New subject: [PATCH 3/6] LAC: implement contents listing and removal by key
---
morphlib/localartifactcache.py | 28 ++++++++++++++++++++++++++++
1 file changed, 28 insertions(+)
diff --git a/morphlib/localartifactcache.py b/morphlib/localartifactcache.py
index b845ceb..d8abf2a 100644
--- a/morphlib/localartifactcache.py
+++ b/morphlib/localartifactcache.py
@@ -14,6 +14,7 @@
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+import collections
import os
import morphlib
@@ -115,3 +116,30 @@ class LocalArtifactCache(object):
for basename in basenames:
os.remove(os.path.join(dirname, basename))
+ def list_contents(self): # pragma: no cover
+ '''Return the set of sources cached and related information.
+
+ returns a [(cache_key, set(artifacts), last_used)]
+
+ '''
+ CacheInfo = collections.namedtuple('CacheInfo', ('artifacts',
'mtime'))
+ contents = collections.defaultdict(lambda: CacheInfo(set(), 0))
+ for dirpath, dirnames, filenames in os.walk(self.cachedir):
+ for filename in filenames:
+ cachekey = filename[:63]
+ artifact = filename[65:]
+ artifacts, max_mtime = contents[cachekey]
+ artifacts.add(artifact)
+ this_mtime = os.stat(os.path.join(dirpath, filename)).st_mtime
+ contents[cachekey] = CacheInfo(artifacts,
+ max(max_mtime, this_mtime))
+ return ((cache_key, info.artifacts, info.mtime)
+ for cache_key, info in contents.iteritems())
+
+
+ def remove(self, cachekey): # pragma: no cover
+ '''Remove all artifacts associated with the given
cachekey.'''
+ for dirpath, dirnames, filenames in os.walk(self.cachedir):
+ for filename in filenames:
+ if filename.startswith(cachekey):
+ os.remove(os.path.join(dirpath, filename))
--
1.7.10.4
10 a.m.
New subject: [PATCH 4/6] lac_tests: Cover list and delete
---
morphlib/localartifactcache.py | 4 ++--
morphlib/localartifactcache_tests.py | 32 ++++++++++++++++++++++++++++++++
2 files changed, 34 insertions(+), 2 deletions(-)
diff --git a/morphlib/localartifactcache.py b/morphlib/localartifactcache.py
index d8abf2a..76d085d 100644
--- a/morphlib/localartifactcache.py
+++ b/morphlib/localartifactcache.py
@@ -116,7 +116,7 @@ class LocalArtifactCache(object):
for basename in basenames:
os.remove(os.path.join(dirname, basename))
- def list_contents(self): # pragma: no cover
+ def list_contents(self):
'''Return the set of sources cached and related information.
returns a [(cache_key, set(artifacts), last_used)]
@@ -137,7 +137,7 @@ class LocalArtifactCache(object):
for cache_key, info in contents.iteritems())
- def remove(self, cachekey): # pragma: no cover
+ def remove(self, cachekey):
'''Remove all artifacts associated with the given
cachekey.'''
for dirpath, dirnames, filenames in os.walk(self.cachedir):
for filename in filenames:
diff --git a/morphlib/localartifactcache_tests.py b/morphlib/localartifactcache_tests.py
index 10f3ab6..082b926 100644
--- a/morphlib/localartifactcache_tests.py
+++ b/morphlib/localartifactcache_tests.py
@@ -157,3 +157,35 @@ class LocalArtifactCacheTests(unittest.TestCase):
cache.clear()
self.assertFalse(cache.has(self.runtime_artifact))
+ def test_put_artifacts_and_list_them_afterwards(self):
+ cache = morphlib.localartifactcache.LocalArtifactCache(
+ self.tempdir.dirname)
+
+ handle = cache.put(self.runtime_artifact)
+ handle.write('runtime')
+ handle.close()
+
+ self.assertTrue(len(list(cache.list_contents())) == 1)
+
+ handle = cache.put(self.devel_artifact)
+ handle.write('devel')
+ handle.close()
+
+ self.assertTrue(len(list(cache.list_contents())) == 1)
+
+ def test_put_artifacts_and_remove_them_afterwards(self):
+ cache = morphlib.localartifactcache.LocalArtifactCache(
+ self.tempdir.dirname)
+
+ handle = cache.put(self.runtime_artifact)
+ handle.write('runtime')
+ handle.close()
+
+ handle = cache.put(self.devel_artifact)
+ handle.write('devel')
+ handle.close()
+
+ key = list(cache.list_contents())[0][0]
+ cache.remove(key)
+
+ self.assertTrue(len(list(cache.list_contents())) == 0)
--
1.7.10.4
10 a.m.
New subject: [PATCH 5/6] plugins: `morph gc` command via plugin
This removes staging areas and extracted chunks from --tempdir.
Then asks the local artifact cache what artifacts it has and how
old they are, removing sources older than the date specified,
oldest first, until there is sufficient space.
---
morphlib/plugins/gc_plugin.py | 122 +++++++++++++++++++++++++++++++++++++++++
without-test-modules | 1 +
2 files changed, 123 insertions(+)
create mode 100644 morphlib/plugins/gc_plugin.py
diff --git a/morphlib/plugins/gc_plugin.py b/morphlib/plugins/gc_plugin.py
new file mode 100644
index 0000000..3445654
--- /dev/null
+++ b/morphlib/plugins/gc_plugin.py
@@ -0,0 +1,122 @@
+# Copyright (C) 2013 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import logging
+import os
+import shutil
+import time
+
+import cliapp
+
+import morphlib
+
+
+class GCPlugin(cliapp.Plugin):
+
+ def enable(self):
+ self.app.add_subcommand('gc', self.gc,
+ arg_synopsis='')
+ self.app.settings.integer(['cachedir-artifact-max-age'],
+ 'how long to keep artifacts around for '
+ 'in seconds, (default: 1 week)',
+ metavar='PERIOD',
+ group="Storage Options",
+ default=(60*60*24*7))
+
+ def disable(self):
+ pass
+
+ def gc(self, args):
+ '''Make space by removing unused files.
+
+ This removes artifacts that have not been used recently,
+ where recently is defined by --cachedir-artifact-max-age.
+
+ This removes extracted chunks and staging areas for failed builds
+ from the directory specified by --tempdir.
+
+ '''
+
+ tempdir = self.app.settings['tempdir']
+ cachedir = self.app.settings['cachedir']
+ tempdir_min_space, cachedir_min_space = \
+ morphlib.util.unify_space_requirements(
+ tempdir, self.app.settings['tempdir-min-space'],
+ cachedir, self.app.settings['cachedir-min-space'])
+
+ self.cleanup_tempdir(tempdir, tempdir_min_space)
+ self.cleanup_cachedir(cachedir, cachedir_min_space)
+
+ def cleanup_tempdir(self, temp_path, min_space):
+ self.app.status(msg='Cleaning up temp dir %(temp_path)s',
+ temp_path=temp_path, chatty=True)
+ for subdir in ('failed', 'chunks'):
+ if morphlib.util.get_bytes_free_in_path(temp_path) >= min_space:
+ self.app.status(msg='Not Removing subdirectory '
+ '%(subdir)s, enough space already cleared',
+ subdir=os.path.join(temp_path, subdir),
+ chatty=True)
+ break
+ self.app.status('Removing temp subdirectory: %(subdir)s',
+ subdir=subdir)
+ path = os.path.join(temp_path, subdir)
+ if os.path.exists(path):
+ shutil.rmtree(path)
+
+ def calculate_latest_deletable_date(self):
+ return time.time() - self.app.settings['cachedir-artifact-max-age']
+
+ def find_deletable_artifacts(self, lac, timestamp):
+ '''Get a list of cache keys in order of how old they
are.'''
+ contents = lac.list_contents()
+ deletable = ((cachekey, mtime)
+ for cachekey, artifacts, mtime in contents
+ if mtime < timestamp)
+ return [cachekey for cachekey, mtime
+ in sorted(deletable, key=lambda x: x[1])]
+
+ def cleanup_cachedir(self, cache_path, min_space):
+ if morphlib.util.get_bytes_free_in_path(cache_path) >= min_space:
+ self.app.status(msg='Not cleaning up cachedir, '
+ 'sufficient space already cleared',
+ chatty=True)
+ return
+ lac = morphlib.localartifactcache.LocalArtifactCache(cache_path)
+ latest_deletable = self.calculate_latest_deletable_date()
+ logging.debug('Can remove artifacts older than timestamp %d'
+ % latest_deletable)
+ deletable = self.find_deletable_artifacts(lac, latest_deletable)
+ logging.debug('Can remove artifacts %s' % repr(deletable))
+ for i, cachekey in enumerate(deletable, start=1):
+ self.app.status(msg='Removing source %(cachekey)s',
+ cachekey=cachekey, chatty=True)
+ lac.remove(cachekey)
+ if morphlib.util.get_bytes_free_in_path(cache_path) >= min_space:
+ self.app.status(msg='Finished cleaning up cachedir with '
+ '%(remaining)d old sources remaining',
+ remaining=(len(deletable) - i), chatty=True)
+ break
+ if morphlib.util.get_bytes_free_in_path(cache_path) >= min_space:
+ self.app.status(msg='Made sufficient space in %(cache_path)s '
+ 'after removing %(sources)d sources',
+ sources=len(deletable), cache_path=cache_path)
+ self.app.status(msg='Unable to clear enough space in %(cache_path)s '
+ 'after removing %(sources)d sources. '
+ 'Please reduce cachedir-artifact-max-age, '
+ 'clear space from elsewhere, enlarge the disk '
+ 'or reduce cachedir-min-space.',
+ cache_path=cache_path, sources=len(deletable),
+ error=True)
diff --git a/without-test-modules b/without-test-modules
index 89b0bd8..f143eb4 100644
--- a/without-test-modules
+++ b/without-test-modules
@@ -26,3 +26,4 @@ morphlib/plugins/__init__.py
morphlib/writeexts.py
morphlib/plugins/copy-artifacts_plugin.py
morphlib/plugins/trovectl_plugin.py
+morphlib/plugins/gc_plugin.py
--
1.7.10.4
10:56 a.m.
New subject: [PATCH 5/6] plugins: `morph gc` command via plugin
On 07/06/13 15:00, Richard Maw wrote:
This removes staging areas and extracted chunks from --tempdir.
Then asks the local artifact cache what artifacts it has and how
old they are, removing sources older than the date specified,
oldest first, until there is sufficient space.
---
morphlib/plugins/gc_plugin.py | 122 +++++++++++++++++++++++++++++++++++++++++
without-test-modules | 1 +
2 files changed, 123 insertions(+)
create mode 100644 morphlib/plugins/gc_plugin.py
diff --git a/morphlib/plugins/gc_plugin.py b/morphlib/plugins/gc_plugin.py
new file mode 100644
index 0000000..3445654
--- /dev/null
+++ b/morphlib/plugins/gc_plugin.py
@@ -0,0 +1,122 @@
+# Copyright (C) 2013 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import logging
+import os
+import shutil
+import time
+
+import cliapp
+
+import morphlib
+
+
+class GCPlugin(cliapp.Plugin):
+
+ def enable(self):
+ self.app.add_subcommand('gc', self.gc,
+ arg_synopsis='')
+ self.app.settings.integer(['cachedir-artifact-max-age'],
+ 'how long to keep artifacts around for '
+ 'in seconds, (default: 1 week)',
+ metavar='PERIOD',
+ group="Storage Options",
+ default=(60*60*24*7))
+
+ def disable(self):
+ pass
+
+ def gc(self, args):
+ '''Make space by removing unused files.
+
+ This removes artifacts that have not been used recently,
+ where recently is defined by --cachedir-artifact-max-age.
+
+ This removes extracted chunks and staging areas for failed builds
+ from the directory specified by --tempdir.
+
+ '''
+
+ tempdir = self.app.settings['tempdir']
+ cachedir = self.app.settings['cachedir']
+ tempdir_min_space, cachedir_min_space = \
+ morphlib.util.unify_space_requirements(
+ tempdir, self.app.settings['tempdir-min-space'],
+ cachedir, self.app.settings['cachedir-min-space'])
+
+ self.cleanup_tempdir(tempdir, tempdir_min_space)
+ self.cleanup_cachedir(cachedir, cachedir_min_space)
+
+ def cleanup_tempdir(self, temp_path, min_space):
+ self.app.status(msg='Cleaning up temp dir %(temp_path)s',
+ temp_path=temp_path, chatty=True)
+ for subdir in ('failed', 'chunks'):
+ if morphlib.util.get_bytes_free_in_path(temp_path) >= min_space:
+ self.app.status(msg='Not Removing subdirectory '
+ '%(subdir)s, enough space already cleared',
+ subdir=os.path.join(temp_path, subdir),
+ chatty=True)
+ break
I think that once a user runs the morph gc command it should
clean everything not necessary, and not stop the cleaning when there is
X of free space. Besides, not cleaning everything increases the probability
that the user will have to run morph gc again in a short period of time.
+ self.app.status('Removing temp subdirectory:
%(subdir)s',
+ subdir=subdir)
+ path = os.path.join(temp_path, subdir)
+ if os.path.exists(path):
+ shutil.rmtree(path)
+
+ def calculate_latest_deletable_date(self):
+ return time.time() - self.app.settings['cachedir-artifact-max-age']
+
+ def find_deletable_artifacts(self, lac, timestamp):
+ '''Get a list of cache keys in order of how old they
are.'''
+ contents = lac.list_contents()
+ deletable = ((cachekey, mtime)
+ for cachekey, artifacts, mtime in contents
+ if mtime < timestamp)
+ return [cachekey for cachekey, mtime
+ in sorted(deletable, key=lambda x: x[1])]
+
+ def cleanup_cachedir(self, cache_path, min_space):
+ if morphlib.util.get_bytes_free_in_path(cache_path) >= min_space:
+ self.app.status(msg='Not cleaning up cachedir, '
+ 'sufficient space already cleared',
+ chatty=True)
+ return
+ lac = morphlib.localartifactcache.LocalArtifactCache(cache_path)
+ latest_deletable = self.calculate_latest_deletable_date()
+ logging.debug('Can remove artifacts older than timestamp %d'
+ % latest_deletable)
+ deletable = self.find_deletable_artifacts(lac, latest_deletable)
+ logging.debug('Can remove artifacts %s' % repr(deletable))
+ for i, cachekey in enumerate(deletable, start=1):
+ self.app.status(msg='Removing source %(cachekey)s',
+ cachekey=cachekey, chatty=True)
+ lac.remove(cachekey)
+ if morphlib.util.get_bytes_free_in_path(cache_path) >= min_space:
+ self.app.status(msg='Finished cleaning up cachedir with '
+ '%(remaining)d old sources remaining',
+ remaining=(len(deletable) - i), chatty=True)
+ break
+ if morphlib.util.get_bytes_free_in_path(cache_path) >= min_space:
+ self.app.status(msg='Made sufficient space in %(cache_path)s '
+ 'after removing %(sources)d sources',
+ sources=len(deletable), cache_path=cache_path)
Same thoughts here. All older than cachedir-artifact-max-age should
be removed.
+ self.app.status(msg='Unable to clear enough space in
%(cache_path)s '
+ 'after removing %(sources)d sources. '
+ 'Please reduce cachedir-artifact-max-age, '
+ 'clear space from elsewhere, enlarge the disk '
+ 'or reduce cachedir-min-space.',
+ cache_path=cache_path, sources=len(deletable),
+ error=True)
diff --git a/without-test-modules b/without-test-modules
index 89b0bd8..f143eb4 100644
--- a/without-test-modules
+++ b/without-test-modules
@@ -26,3 +26,4 @@ morphlib/plugins/__init__.py
morphlib/writeexts.py
morphlib/plugins/copy-artifacts_plugin.py
morphlib/plugins/trovectl_plugin.py
+morphlib/plugins/gc_plugin.py
10:58 a.m.
New subject: [PATCH 5/6] plugins: `morph gc` command via plugin
On 07/06/2013 15:56, Tiago Gomes wrote:
I think that once a user runs the morph gc command it should
clean everything not necessary, and not stop the cleaning when there is
X of free space. Besides, not cleaning everything increases the probability
that the user will have to run morph gc again in a short period of time.
+1
11:17 a.m.
New subject: [PATCH 5/6] plugins: `morph gc` command via plugin
On Fri, Jun 07, 2013 at 03:58:03PM +0100, Paul Sherwood wrote:
On 07/06/2013 15:56, Tiago Gomes wrote:
> I think that once a user runs the morph gc command it should
> clean everything not necessary, and not stop the cleaning when there is
> X of free space. Besides, not cleaning everything increases the probability
> that the user will have to run morph gc again in a short period of time.
+1
I disagree on the grounds that the aim is to reduce cycle time. Old
artifacts may still be needed later, and it takes longer to rebuild an
artifact than be notified you don't have enough space and run another gc.
I'll change it if people still think everything should be removed,
but everything has been cached for a reason.
11:23 a.m.
New subject: [PATCH 5/6] plugins: `morph gc` command via plugin
On 07/06/2013 16:17, Richard Maw wrote:
On Fri, Jun 07, 2013 at 03:58:03PM +0100, Paul Sherwood wrote:
> On 07/06/2013 15:56, Tiago Gomes wrote:
>> I think that once a user runs the morph gc command it should
>> clean everything not necessary, and not stop the cleaning when there is
>> X of free space. Besides, not cleaning everything increases the probability
>> that the user will have to run morph gc again in a short period of time.
>
> +1
I disagree on the grounds that the aim is to reduce cycle time. Old
artifacts may still be needed later, and it takes longer to rebuild an
artifact than be notified you don't have enough space and run another gc.
I'll change it if people still think everything should be removed,
but everything has been cached for a reason.
interesting dilemma. yet another setting for morph conf?
11:28 a.m.
New subject: [PATCH 5/6] plugins: `morph gc` command via plugin
On Fri, Jun 07, 2013 at 04:23:54PM +0100, Paul Sherwood wrote:
> I disagree on the grounds that the aim is to reduce cycle time.
Old
> artifacts may still be needed later, and it takes longer to rebuild an
> artifact than be notified you don't have enough space and run another gc.
>
> I'll change it if people still think everything should be removed,
> but everything has been cached for a reason.
interesting dilemma. yet another setting for morph conf?
I'd rather not have a proliferation of these settings.
However, rather than just a flag, how about two periods, so you'd effectively
have a keep-younger-than and delete-older-than, so when you gc it always gets
rid of those older than the specified date, and it may delete newer than that
if it still needs to make space?
I will point out that such functionality is unlikely to be ready for review
by the end of today and I'm on holiday next week. I'll see if I can hack it
up quickly.
1:04 p.m.
New subject: [PATCHv3 0/6] Add morph gc subcommand
Repo: git://git.baserock.org/baserock/baserock/morph.git
Ref: baserock/richardmaw/S7905-morph-gc-v3
Sha1: f461727eeb43bdf1e9caf385997fd9fc860d8e1a
Land: master
I knocked this together in half an hour. This offers always
deleting artifacts older than a week and possibly deleting
anything older than a day.
If this is the wrong approach feel free to do something different, but I
won't be working on it for a week, due to going on holiday.
Richard Maw (6):
util: split out space unify logic
lac_tests: set cache key
LAC: implement contents listing and removal by key
lac_tests: Cover list and delete
plugins: add morph gc subcommand
util: suggest how to deal with insufficient space
morphlib/localartifactcache.py | 28 ++++++
morphlib/localartifactcache_tests.py | 36 +++++++-
morphlib/plugins/gc_plugin.py | 158 ++++++++++++++++++++++++++++++++++
morphlib/util.py | 33 +++++--
without-test-modules | 1 +
5 files changed, 248 insertions(+), 8 deletions(-)
create mode 100644 morphlib/plugins/gc_plugin.py
--
1.7.10.4
1:04 p.m.
New subject: [PATCH 1/6] util: split out space unify logic
---
morphlib/util.py | 25 +++++++++++++++++++------
1 file changed, 19 insertions(+), 6 deletions(-)
diff --git a/morphlib/util.py b/morphlib/util.py
index 1a347da..6dcb22b 100644
--- a/morphlib/util.py
+++ b/morphlib/util.py
@@ -237,16 +237,29 @@ def on_same_filesystem(path_a, path_b): # pragma: no cover
# TODO: return true if one path is a subvolume of the other on btrfs?
return os.stat(path_a).st_dev == os.stat(path_b).st_dev
+def unify_space_requirements(tmp_path, tmp_min_size,
+ cache_path, cache_min_size): # pragma: no cover
+ """Adjust minimum sizes when paths share a disk.
+
+ Given pairs of path and minimum size, return the minimum sizes such
+ that when the paths are on the same disk, the sizes are added together.
+
+ """
+ # TODO: make this work for variable number of (path, size) pairs as needed
+ # hint: try list.sort and itertools.groupby
+ if not on_same_filesystem(tmp_path, cache_path):
+ return tmp_min_size, cache_min_size
+ unified_size = tmp_min_size + cache_min_size
+ return unified_size, unified_size
+
def check_disk_available(tmp_path, tmp_min_size,
cache_path, cache_min_size): # pragma: no cover
# if both are on the same filesystem, assume they share a storage pool,
# so the sum of the two sizes needs to be available
- # TODO: if we need to do this on any more than 2 filesystems
- # extend it to take a [(path, min)] and do some arcane mathematics
- # to split it into groups that share a filesystem
- # hint: try list.sort and itertools.groupby
- if on_same_filesystem(tmp_path, cache_path):
- tmp_min_size = cache_min_size = tmp_min_size + cache_min_size
+ # TODO: if we need to do this on any more than 2 paths
+ # extend it to take a [(path, min)]
+ tmp_min_size, cache_min_size = unify_space_requirements(
+ tmp_path, tmp_min_size, cache_path, cache_min_size)
tmp_size, cache_size = map(get_bytes_free_in_path, (tmp_path, cache_path))
errors = []
for path, min in [(tmp_path, tmp_min_size), (cache_path, cache_min_size)]:
--
1.7.10.4
1:04 p.m.
New subject: [PATCH 2/6] lac_tests: set cache key
---
morphlib/localartifactcache_tests.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/morphlib/localartifactcache_tests.py b/morphlib/localartifactcache_tests.py
index 36b5e89..10f3ab6 100644
--- a/morphlib/localartifactcache_tests.py
+++ b/morphlib/localartifactcache_tests.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012 Codethink Limited
+# Copyright (C) 2012,2013 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -47,8 +47,10 @@ class LocalArtifactCacheTests(unittest.TestCase):
'repo', 'ref', 'sha1', 'tree', morph,
'chunk.morph')
self.runtime_artifact = morphlib.artifact.Artifact(
self.source, 'chunk-runtime')
+ self.runtime_artifact.cache_key = '0'*64
self.devel_artifact = morphlib.artifact.Artifact(
self.source, 'chunk-devel')
+ self.devel_artifact.cache_key = '0'*64
def tearDown(self):
self.tempdir.remove()
--
1.7.10.4
1:04 p.m.
New subject: [PATCH 3/6] LAC: implement contents listing and removal by key
---
morphlib/localartifactcache.py | 28 ++++++++++++++++++++++++++++
1 file changed, 28 insertions(+)
diff --git a/morphlib/localartifactcache.py b/morphlib/localartifactcache.py
index b845ceb..d8abf2a 100644
--- a/morphlib/localartifactcache.py
+++ b/morphlib/localartifactcache.py
@@ -14,6 +14,7 @@
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+import collections
import os
import morphlib
@@ -115,3 +116,30 @@ class LocalArtifactCache(object):
for basename in basenames:
os.remove(os.path.join(dirname, basename))
+ def list_contents(self): # pragma: no cover
+ '''Return the set of sources cached and related information.
+
+ returns a [(cache_key, set(artifacts), last_used)]
+
+ '''
+ CacheInfo = collections.namedtuple('CacheInfo', ('artifacts',
'mtime'))
+ contents = collections.defaultdict(lambda: CacheInfo(set(), 0))
+ for dirpath, dirnames, filenames in os.walk(self.cachedir):
+ for filename in filenames:
+ cachekey = filename[:63]
+ artifact = filename[65:]
+ artifacts, max_mtime = contents[cachekey]
+ artifacts.add(artifact)
+ this_mtime = os.stat(os.path.join(dirpath, filename)).st_mtime
+ contents[cachekey] = CacheInfo(artifacts,
+ max(max_mtime, this_mtime))
+ return ((cache_key, info.artifacts, info.mtime)
+ for cache_key, info in contents.iteritems())
+
+
+ def remove(self, cachekey): # pragma: no cover
+ '''Remove all artifacts associated with the given
cachekey.'''
+ for dirpath, dirnames, filenames in os.walk(self.cachedir):
+ for filename in filenames:
+ if filename.startswith(cachekey):
+ os.remove(os.path.join(dirpath, filename))
--
1.7.10.4
1:04 p.m.
New subject: [PATCH 4/6] lac_tests: Cover list and delete
---
morphlib/localartifactcache.py | 4 ++--
morphlib/localartifactcache_tests.py | 32 ++++++++++++++++++++++++++++++++
2 files changed, 34 insertions(+), 2 deletions(-)
diff --git a/morphlib/localartifactcache.py b/morphlib/localartifactcache.py
index d8abf2a..76d085d 100644
--- a/morphlib/localartifactcache.py
+++ b/morphlib/localartifactcache.py
@@ -116,7 +116,7 @@ class LocalArtifactCache(object):
for basename in basenames:
os.remove(os.path.join(dirname, basename))
- def list_contents(self): # pragma: no cover
+ def list_contents(self):
'''Return the set of sources cached and related information.
returns a [(cache_key, set(artifacts), last_used)]
@@ -137,7 +137,7 @@ class LocalArtifactCache(object):
for cache_key, info in contents.iteritems())
- def remove(self, cachekey): # pragma: no cover
+ def remove(self, cachekey):
'''Remove all artifacts associated with the given
cachekey.'''
for dirpath, dirnames, filenames in os.walk(self.cachedir):
for filename in filenames:
diff --git a/morphlib/localartifactcache_tests.py b/morphlib/localartifactcache_tests.py
index 10f3ab6..082b926 100644
--- a/morphlib/localartifactcache_tests.py
+++ b/morphlib/localartifactcache_tests.py
@@ -157,3 +157,35 @@ class LocalArtifactCacheTests(unittest.TestCase):
cache.clear()
self.assertFalse(cache.has(self.runtime_artifact))
+ def test_put_artifacts_and_list_them_afterwards(self):
+ cache = morphlib.localartifactcache.LocalArtifactCache(
+ self.tempdir.dirname)
+
+ handle = cache.put(self.runtime_artifact)
+ handle.write('runtime')
+ handle.close()
+
+ self.assertTrue(len(list(cache.list_contents())) == 1)
+
+ handle = cache.put(self.devel_artifact)
+ handle.write('devel')
+ handle.close()
+
+ self.assertTrue(len(list(cache.list_contents())) == 1)
+
+ def test_put_artifacts_and_remove_them_afterwards(self):
+ cache = morphlib.localartifactcache.LocalArtifactCache(
+ self.tempdir.dirname)
+
+ handle = cache.put(self.runtime_artifact)
+ handle.write('runtime')
+ handle.close()
+
+ handle = cache.put(self.devel_artifact)
+ handle.write('devel')
+ handle.close()
+
+ key = list(cache.list_contents())[0][0]
+ cache.remove(key)
+
+ self.assertTrue(len(list(cache.list_contents())) == 0)
--
1.7.10.4
1:04 p.m.
New subject: [PATCH 5/6] plugins: add morph gc subcommand
This removes staging areas and extracted chunks from --tempdir.
Then asks the local artifact cache what artifacts it
has and how old they are, removing all sources older than
--cachedir-artifact-delete-older-than, and may delete other sources that
are younger than --cachedir-artifact-keep-younger-than if it still needs
to make space.
---
morphlib/plugins/gc_plugin.py | 158 +++++++++++++++++++++++++++++++++++++++++
without-test-modules | 1 +
2 files changed, 159 insertions(+)
create mode 100644 morphlib/plugins/gc_plugin.py
diff --git a/morphlib/plugins/gc_plugin.py b/morphlib/plugins/gc_plugin.py
new file mode 100644
index 0000000..cc82cae
--- /dev/null
+++ b/morphlib/plugins/gc_plugin.py
@@ -0,0 +1,158 @@
+# Copyright (C) 2013 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import logging
+import os
+import shutil
+import time
+
+import cliapp
+
+import morphlib
+
+
+class GCPlugin(cliapp.Plugin):
+
+ def enable(self):
+ self.app.add_subcommand('gc', self.gc,
+ arg_synopsis='')
+ self.app.settings.integer(['cachedir-artifact-delete-older-than'],
+ 'always delete artifacts older than this '
+ 'period in seconds, (default: 1 week)',
+ metavar='PERIOD',
+ group="Storage Options",
+ default=(60*60*24*7))
+ self.app.settings.integer(['cachedir-artifact-keep-younger-than'],
+ 'allow deletion of artifacts older than '
+ 'this period in seconds, (default: 1 day)',
+ metavar='PERIOD',
+ group="Storage Options",
+ default=(60*60*24))
+
+ def disable(self):
+ pass
+
+ def gc(self, args):
+ '''Make space by removing unused files.
+
+ This removes all artifacts older than
+ --cachedir-artifact-delete-older-than, and may delete artifacts
+ older than --cachedir-artifact-keep-younger-than if it still
+ needs to make space.
+
+ This removes extracted chunks and staging areas for failed builds
+ from the directory specified by --tempdir.
+
+ '''
+
+ tempdir = self.app.settings['tempdir']
+ cachedir = self.app.settings['cachedir']
+ tempdir_min_space, cachedir_min_space = \
+ morphlib.util.unify_space_requirements(
+ tempdir, self.app.settings['tempdir-min-space'],
+ cachedir, self.app.settings['cachedir-min-space'])
+
+ self.cleanup_tempdir(tempdir, tempdir_min_space)
+ self.cleanup_cachedir(cachedir, cachedir_min_space)
+
+ def cleanup_tempdir(self, temp_path, min_space):
+ self.app.status(msg='Cleaning up temp dir %(temp_path)s',
+ temp_path=temp_path, chatty=True)
+ for subdir in ('failed', 'chunks'):
+ if morphlib.util.get_bytes_free_in_path(temp_path) >= min_space:
+ self.app.status(msg='Not Removing subdirectory '
+ '%(subdir)s, enough space already cleared',
+ subdir=os.path.join(temp_path, subdir),
+ chatty=True)
+ break
+ self.app.status('Removing temp subdirectory: %(subdir)s',
+ subdir=subdir)
+ path = os.path.join(temp_path, subdir)
+ if os.path.exists(path):
+ shutil.rmtree(path)
+
+ def calculate_delete_range(self):
+ now = time.time()
+ always_delete_age = \
+ now - self.app.settings['cachedir-artifact-delete-older-than']
+ may_delete_age = \
+ now - self.app.settings['cachedir-artifact-keep-younger-than']
+ return always_delete_age, may_delete_age
+
+ def find_deletable_artifacts(self, lac, max_age, min_age):
+ '''Get a list of cache keys in order of how old they
are.'''
+ contents = list(lac.list_contents())
+ always = set(cachekey
+ for cachekey, artifacts, mtime in contents
+ if mtime < max_age)
+ maybe = ((cachekey, mtime)
+ for cachekey, artifacts, mtime in contents
+ if max_age <= mtime < min_age)
+ return always, [cachekey for cachekey, mtime
+ in sorted(maybe, key=lambda x: x[1])]
+
+ def cleanup_cachedir(self, cache_path, min_space):
+ def sufficient_free():
+ free = morphlib.util.get_bytes_free_in_path(cache_path)
+ return (free >= min_space)
+ if sufficient_free():
+ self.app.status(msg='Not cleaning up cachedir, '
+ 'sufficient space already cleared',
+ chatty=True)
+ return
+ lac = morphlib.localartifactcache.LocalArtifactCache(cache_path)
+ max_age, min_age = self.calculate_delete_range()
+ logging.debug('Must remove artifacts older than timestamp %d'
+ % max_age)
+ always_delete, may_delete = \
+ self.find_deletable_artifacts(lac, max_age, min_age)
+ removed = 0
+ source_count = len(always_delete) + len(may_delete)
+ logging.debug('Must remove artifacts %s' % repr(always_delete))
+ logging.debug('Can remove artifacts %s' % repr(may_delete))
+
+ # Remove all old artifacts
+ for cachekey in always_delete:
+ self.app.status(msg='Removing source %(cachekey)s',
+ cachekey=cachekey, chatty=True)
+ lac.remove(cachekey)
+ removed += 1
+
+ # Maybe remove remaining middle-aged artifacts
+ for cachekey in may_delete:
+ if sufficient_free():
+ self.app.status(msg='Finished cleaning up cachedir with '
+ '%(remaining)d old sources remaining',
+ remaining=(source_count - removed),
+ chatty=True)
+ break
+ self.app.status(msg='Removing source %(cachekey)s',
+ cachekey=cachekey, chatty=True)
+ lac.remove(cachekey)
+ removed += 1
+
+ if sufficient_free():
+ self.app.status(msg='Made sufficient space in %(cache_path)s '
+ 'after removing %(removed)d sources',
+ removed=removed, cache_path=cache_path)
+ return
+ self.app.status(msg='Unable to clear enough space in %(cache_path)s '
+ 'after removing %(removed)d sources. Please '
+ 'reduce cachedir-artifact-keep-younger-than, '
+ 'clear space from elsewhere, enlarge the disk '
+ 'or reduce cachedir-min-space.',
+ cache_path=cache_path, removed=removed,
+ error=True)
diff --git a/without-test-modules b/without-test-modules
index 89b0bd8..f143eb4 100644
--- a/without-test-modules
+++ b/without-test-modules
@@ -26,3 +26,4 @@ morphlib/plugins/__init__.py
morphlib/writeexts.py
morphlib/plugins/copy-artifacts_plugin.py
morphlib/plugins/trovectl_plugin.py
+morphlib/plugins/gc_plugin.py
--
1.7.10.4
9:43 a.m.
New subject: [PATCH 5/6] plugins: add morph gc subcommand
On 07/06/13 18:04, Richard Maw wrote:
This removes staging areas and extracted chunks from --tempdir.
Then asks the local artifact cache what artifacts it
has and how old they are, removing all sources older than
--cachedir-artifact-delete-older-than, and may delete other sources that
are younger than --cachedir-artifact-keep-younger-than if it still needs
to make space.
---
morphlib/plugins/gc_plugin.py | 158 +++++++++++++++++++++++++++++++++++++++++
without-test-modules | 1 +
2 files changed, 159 insertions(+)
create mode 100644 morphlib/plugins/gc_plugin.py
diff --git a/morphlib/plugins/gc_plugin.py b/morphlib/plugins/gc_plugin.py
new file mode 100644
index 0000000..cc82cae
--- /dev/null
+++ b/morphlib/plugins/gc_plugin.py
@@ -0,0 +1,158 @@
+# Copyright (C) 2013 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import logging
+import os
+import shutil
+import time
+
+import cliapp
+
+import morphlib
+
+
+class GCPlugin(cliapp.Plugin):
+
+ def enable(self):
+ self.app.add_subcommand('gc', self.gc,
+ arg_synopsis='')
+ self.app.settings.integer(['cachedir-artifact-delete-older-than'],
+ 'always delete artifacts older than this '
+ 'period in seconds, (default: 1 week)',
+ metavar='PERIOD',
+ group="Storage Options",
+ default=(60*60*24*7))
+ self.app.settings.integer(['cachedir-artifact-keep-younger-than'],
+ 'allow deletion of artifacts older than '
+ 'this period in seconds, (default: 1 day)',
+ metavar='PERIOD',
+ group="Storage Options",
+ default=(60*60*24))
I wonder if we really need those two settings. Can't we simple order the
cache
artifact list by modification date, and then keep deleting artifacts
until we had
freed enough disk space (the free disk space on filesystem where
cachedir is,
be higher than self.app.settings['cachedir-min-space']).
Or keep just one setting and remove all artifacts older than this setting.
And if we still don’t have not enough space, we keep walk the list of the
remaining artifacts and delete them until we have enough free space.
+
+ def disable(self):
+ pass
+
+ def gc(self, args):
+ '''Make space by removing unused files.
+
+ This removes all artifacts older than
+ --cachedir-artifact-delete-older-than, and may delete artifacts
+ older than --cachedir-artifact-keep-younger-than if it still
+ needs to make space.
+
+ This removes extracted chunks and staging areas for failed builds
+ from the directory specified by --tempdir.
+
+ '''
+
+ tempdir = self.app.settings['tempdir']
+ cachedir = self.app.settings['cachedir']
+ tempdir_min_space, cachedir_min_space = \
+ morphlib.util.unify_space_requirements(
+ tempdir, self.app.settings['tempdir-min-space'],
+ cachedir, self.app.settings['cachedir-min-space'])
+
+ self.cleanup_tempdir(tempdir, tempdir_min_space)
+ self.cleanup_cachedir(cachedir, cachedir_min_space)
+
+ def cleanup_tempdir(self, temp_path, min_space):
+ self.app.status(msg='Cleaning up temp dir %(temp_path)s',
+ temp_path=temp_path, chatty=True)
+ for subdir in ('failed', 'chunks'):
I think we should also clean staging and deployments. It is true
that failed builds are moved to the failed dir, but that is not the
case if morph crashes due an unexpected error. And when we
are hacking on morph this happens occasionally. Deployments
should also be cleaned because currently we don't remove failed
deployments in all configuration extensions.
+ if morphlib.util.get_bytes_free_in_path(temp_path) >=
min_space:
+ self.app.status(msg='Not Removing subdirectory '
+ '%(subdir)s, enough space already cleared',
+ subdir=os.path.join(temp_path, subdir),
+ chatty=True)
+ break
+ self.app.status('Removing temp subdirectory: %(subdir)s',
+ subdir=subdir)
+ path = os.path.join(temp_path, subdir)
+ if os.path.exists(path):
+ shutil.rmtree(path)
+
+ def calculate_delete_range(self):
+ now = time.time()
+ always_delete_age = \
+ now - self.app.settings['cachedir-artifact-delete-older-than']
+ may_delete_age = \
+ now - self.app.settings['cachedir-artifact-keep-younger-than']
+ return always_delete_age, may_delete_age
+
+ def find_deletable_artifacts(self, lac, max_age, min_age):
+ '''Get a list of cache keys in order of how old they
are.'''
+ contents = list(lac.list_contents())
+ always = set(cachekey
+ for cachekey, artifacts, mtime in contents
+ if mtime < max_age)
+ maybe = ((cachekey, mtime)
+ for cachekey, artifacts, mtime in contents
+ if max_age <= mtime < min_age)
+ return always, [cachekey for cachekey, mtime
+ in sorted(maybe, key=lambda x: x[1])]
+
+ def cleanup_cachedir(self, cache_path, min_space):
+ def sufficient_free():
+ free = morphlib.util.get_bytes_free_in_path(cache_path)
+ return (free >= min_space)
+ if sufficient_free():
+ self.app.status(msg='Not cleaning up cachedir, '
+ 'sufficient space already cleared',
+ chatty=True)
+ return
+ lac = morphlib.localartifactcache.LocalArtifactCache(cache_path)
+ max_age, min_age = self.calculate_delete_range()
+ logging.debug('Must remove artifacts older than timestamp %d'
+ % max_age)
+ always_delete, may_delete = \
+ self.find_deletable_artifacts(lac, max_age, min_age)
+ removed = 0
+ source_count = len(always_delete) + len(may_delete)
+ logging.debug('Must remove artifacts %s' % repr(always_delete))
+ logging.debug('Can remove artifacts %s' % repr(may_delete))
+
+ # Remove all old artifacts
+ for cachekey in always_delete:
+ self.app.status(msg='Removing source %(cachekey)s',
+ cachekey=cachekey, chatty=True)
+ lac.remove(cachekey)
+ removed += 1
+
+ # Maybe remove remaining middle-aged artifacts
+ for cachekey in may_delete:
+ if sufficient_free():
+ self.app.status(msg='Finished cleaning up cachedir with '
+ '%(remaining)d old sources remaining',
+ remaining=(source_count - removed),
+ chatty=True)
+ break
+ self.app.status(msg='Removing source %(cachekey)s',
+ cachekey=cachekey, chatty=True)
+ lac.remove(cachekey)
+ removed += 1
+
+ if sufficient_free():
+ self.app.status(msg='Made sufficient space in %(cache_path)s '
+ 'after removing %(removed)d sources',
+ removed=removed, cache_path=cache_path)
+ return
+ self.app.status(msg='Unable to clear enough space in %(cache_path)s '
+ 'after removing %(removed)d sources. Please '
+ 'reduce cachedir-artifact-keep-younger-than, '
+ 'clear space from elsewhere, enlarge the disk '
+ 'or reduce cachedir-min-space.',
+ cache_path=cache_path, removed=removed,
+ error=True)
diff --git a/without-test-modules b/without-test-modules
index 89b0bd8..f143eb4 100644
--- a/without-test-modules
+++ b/without-test-modules
@@ -26,3 +26,4 @@ morphlib/plugins/__init__.py
morphlib/writeexts.py
morphlib/plugins/copy-artifacts_plugin.py
morphlib/plugins/trovectl_plugin.py
+morphlib/plugins/gc_plugin.py
5:20 a.m.
New subject: [PATCH 5/6] plugins: add morph gc subcommand
On Tue, Jun 11, 2013 at 02:43:21PM +0100, Tiago Gomes wrote:
On 07/06/13 18:04, Richard Maw wrote:
<snip>
>+
self.app.settings.integer(['cachedir-artifact-delete-older-than'],
>+ 'always delete artifacts older than this
'
>+ 'period in seconds, (default: 1 week)',
>+ metavar='PERIOD',
>+ group="Storage Options",
>+ default=(60*60*24*7))
>+ self.app.settings.integer(['cachedir-artifact-keep-younger-than'],
>+ 'allow deletion of artifacts older than '
>+ 'this period in seconds, (default: 1
day)',
>+ metavar='PERIOD',
>+ group="Storage Options",
>+ default=(60*60*24))
I wonder if we really need those two settings. Can't we simple order
the cache
artifact list by modification date, and then keep deleting artifacts
until we had
freed enough disk space (the free disk space on filesystem where
cachedir is,
be higher than self.app.settings['cachedir-min-space']).
Or keep just one setting and remove all artifacts older than this setting.
And if we still don’t have not enough space, we keep walk the list of the
remaining artifacts and delete them until we have enough free space.
Deleting everything until you reach the minimum space is not the only way to
ensure you have enough space. You may have your minimum space setting too high,
or you may have accidentally put something else on the disk.
I would not like to lose hours of build time to this accident, so I think it's
valuable to have a cut-off.
Likewise, artifacts older than a certain date are unlikely to be used again,
so it's worth getting rid of all of them, instead of just removing them
as-needed, since filesystems behave better with more free space.
<snip>
>+ def cleanup_tempdir(self, temp_path, min_space):
>+ self.app.status(msg='Cleaning up temp dir %(temp_path)s',
>+ temp_path=temp_path, chatty=True)
>+ for subdir in ('failed', 'chunks'):
I think we should also clean staging and deployments. It is true
that failed builds are moved to the failed dir, but that is not the
case if morph crashes due an unexpected error. And when we
are hacking on morph this happens occasionally. Deployments
should also be cleaned because currently we don't remove failed
deployments in all configuration extensions.
The use-case for this plugin is generally for end-user developers, not morph
developers. I'd expect morph developers to be able to clean up themselves in
the case of internal errors.
I'm not sure I fully understand your point about failed deployments though,
I thought morph removed the extracted deployment directory.
They may not clean up their temporary files though. I'd have to
investigate further.
1:04 p.m.
New subject: [PATCH 6/6] util: suggest how to deal with insufficient space
---
morphlib/util.py | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/morphlib/util.py b/morphlib/util.py
index 6dcb22b..b56443e 100644
--- a/morphlib/util.py
+++ b/morphlib/util.py
@@ -269,4 +269,10 @@ def check_disk_available(tmp_path, tmp_min_size,
'has %(free)d' % locals())
if not errors:
return
- raise morphlib.Error('Insufficient space on disk:\n' +
'\n'.join(errors))
+ raise morphlib.Error('Insufficient space on disk:\n' +
+ '\n'.join(errors) + '\n'
+ 'Please run `morph gc`. If the problem persists '
+ 'increase the disk size, manually clean up some '
+ 'space or reduce the disk space required by the '
+ 'tempdir-min-space and cachedir-min-space '
+ 'configuration options.')
--
1.7.10.4
5:55 a.m.
New subject: Merged: [PATCHv3 0/6] Add morph gc subcommand
On Fri, Jun 07, 2013 at 06:04:03PM +0100, Richard Maw wrote:
Repo: git://git.baserock.org/baserock/baserock/morph.git
Ref: baserock/richardmaw/S7905-morph-gc-v3
Sha1: f461727eeb43bdf1e9caf385997fd9fc860d8e1a
Land: master
There were no major objections to the design and it has been sitting in
the merge queue long enough that it was decided to merge it.
It has been reviewed and lightly tested by Jonathan Maw.
2:35 p.m.
New subject: Merged: [PATCHv3 0/6] Add morph gc subcommand
On 2013-06-19 10:55, Richard Maw wrote:
On Fri, Jun 07, 2013 at 06:04:03PM +0100, Richard Maw wrote:
> Repo: git://git.baserock.org/baserock/baserock/morph.git
> Ref: baserock/richardmaw/S7905-morph-gc-v3
> Sha1: f461727eeb43bdf1e9caf385997fd9fc860d8e1a
> Land: master
There were no major objections to the design and it has been sitting
in
the merge queue long enough that it was decided to merge it.
It has been reviewed and lightly tested by Jonathan Maw
I ran 'morph gc'. It reported:
2013-06-22 22:11:16 Removing temp subdirectory: failed
2013-06-22 22:11:16 Removing temp subdirectory: chunks
My /src filesystem usage according to df dropped from 84% to 83%.
There's still a load of stuff in /src/tmp, for example.
I guess I must be doing it wrong.
btw morph --version gives baserock-8-67-g155e502-unreproducible
5:37 a.m.
New subject: Merged: [PATCHv3 0/6] Add morph gc subcommand
On Sun, Jun 23, 2013 at 07:35:01PM +0100, Paul Sherwood wrote:
I ran 'morph gc'. It reported:
2013-06-22 22:11:16 Removing temp subdirectory: failed
2013-06-22 22:11:16 Removing temp subdirectory: chunks
Sorry about that first one, it looks like an error message rather than
status that it's removing stuff from the failed builds directory.
My /src filesystem usage according to df dropped from 84% to 83%.
There's still a load of stuff in /src/tmp, for example.
I guess I must be doing it wrong.
btw morph --version gives baserock-8-67-g155e502-unreproducible
Could you be more specific about what's still left?
Part of the change was to put the different uses for the tempdir
into different subdirectories, so old chunks and staging areas from
before the gc code was merged will still be around.
We didn't anticipate this change at the time, so currently as part
of the transision the extracted chunks and staging areas have to be
removed manually.
7:37 a.m.
New subject: Merged: [PATCHv3 0/6] Add morph gc subcommand
On 2013-06-24 10:37, Richard Maw wrote:
Could you be more specific about what's still left?
I re-ran it again -
2013-06-23 05:28:17 Removing temp subdirectory: failed
2013-06-23 05:28:17 Removing temp subdirectory: chunks
2013-06-23 05:28:20 Made sufficient space in /src/cache after removing
86 sources
I suggest that last message should just be "Removed 86 sources from
/src/cache"
Still lots of stuff in /src/tmp. morph (see below)
gc should just *clear* /src/tmp, surely
001b16071e0d99036497ff519b83e95df275d4ba72f1d1374ed2715dc93d7396.chunk.curl.d
0146c5dd0c57ef24883e8b1a05edc53175bc717e2dd3c689f98e2a5a81c862cc.chunk.xorg-app-xauth.d
015f4a71af885782b21f59edf62b10e3361cf1a83c6de12014ff637b9bb6f18d.chunk.xorg-lib-libXmu.d
05aeb6b96e7ad76523cfb033b158187af966075b78bd1425e0625b8fec775c9a.chunk.automake.d
07198b4c57c1249d72ef1cf5cb482cf4529a343cfa25824b3eaf9fdf11f75010.chunk.xorg-lib-libICE.d
07e8b989f8f7655e6188ea94642b06ba8b22849b15b35f2a2912f15b1b716eff.chunk.xorg-lib-libdmx.d
09d2d1b6c78e0c707b5aebc56cdef525e803a7c678859637aa3ffd59f82aea12.chunk.ncurses.d
0a19aaf5acc28eef816cdee36915d59a67599ffdd68200d8db96d044594d91bc.chunk.xorg-proto-renderproto.d
0ce5da6889043d63891a5670b0762e10ec9ccc1b5a4094a18fadc8b94829fd9b.chunk.gobject-introspection.d
0f33f7e9037fa3e2393ed62a98e0415971728d0cf55dddb3afd86e6e72ce86f0.chunk.xorg-proto-fixesproto.d
10e49b1c729c7b51ecbdbd68b0a8731b4da0afd33460f34c771efba844701050.chunk.xorg-lib-libpciaccess.d
123def626b5e72ee2861658732a4cb4c063a398f8a3f0fb8bce0b45e0610f78a.chunk.zlib.d
151d17d7dd2cbd31c4f216477d46e54b3f478b84441130a996e6d301a08e0d2d.chunk.linux-api-headers.d
17e9f5b76982155d2dd464556b0b801078335effd7582a5a01d60946aca18be5.chunk.xorg-lib-libXcursor.d
186cfec3a13e51d4afd8ede617abe316718670860f3c14968f3b851cc58d881b.chunk.libusbx.d
1989d9002613e33c26608e27a63c4a1d15f9daf532ff348e6e0489d1496809e1.chunk.dbus.d
1b1e2dcde3619f80c9ee69ede3a94715da41116570c2aaf27de29b1aa1ce0c21.chunk.make.d
1b1f3de6952f82edd8eb01b4531bbee93fc586e2c28ddcd7c2a0fade221306ca.chunk.texinfo-tarball.d
1b53f3bcb256303adbbdfad36c798b08c532a2cf64255b16bff19a12618eb8df.chunk.xorg-proto-dri2proto.d
1d89f5cce8afb94811b5210c46e8c3079f754ba4c5f635f32dd514f49397ccd5.chunk.cpython.d
1dc2c053fd53db35dc250d36721fe629b1b424d780e54942db9d071b5a6e7514.chunk.xkeyboard-config.d
1e3b13e1827902771dc7d8af033908be01c7a231cbc93df7b6872734024d5231.chunk.xorg-lib-libXinerama.d
1e69962ad51af7c6f40c44f883c5812a3cdb84b5a5a23b85f150027271b72341.chunk.xorg-proto-printproto.d
1e7ae374b7d42897f870f52389e8bfbb1abca05f7eb4011b2ebf7d6b916fce67.chunk.busybox.d
20d8cbc5c841456d05106612b97655677e794f68c92b9c1ecba7c12260f01079.chunk.xcb-proto.d
21fad446f43070fc2041ebd2c749ebdcaa541196b0bf0113215547c835196096.chunk.fuse.d
2319a4ace1fc2e036ddb08d25f04f641ebb9dfcedcc842687d07596131b99ec4.chunk.XML-Parser.d
2424395d5fc14ee6ed7521ec7fe03db674a576688bafe92854d6199e3955027c.chunk.xorg-lib-libXau.d
24e8a3a7ab0d1db5924d737d931b1998875c916459cec8e58368133bd56c931c.chunk.xorg-proto-xf86bigfontproto.d
300838ae1f50d5a9121c413ec089a95ddde67cd67d29df8bf99dc2b7d428450a.chunk.bison.d
31907529cb51c332301d128fa514c12695472b4090ed449c61aadf49dc4c0532.chunk.nano.d
32c66dfd4168cb88b87a5f6462186d4ffd33e3b65227a5e0e365b1fa54757c83.chunk.xorg-proto-xf86driproto.d
3542b2b9e9f83cc8b35da6ea453cbda0403231917e6e69e208c7238b17e6b100.chunk.fhs-dirs.d
35ccce48f93dbecc5670cb353c0e7eb3c8b4c250959de8c068be2a4a1a5a0536.chunk.help2man.d
35e7ccf68ca63967eafa0f2538f1cd02faacf78ddfce4dad0b50dbbf10a675f9.chunk.xorg-proto-resourceproto.d
381946f5ec3525a3ef17f18e1533bd7a815544617cfde5d9abfdf386f0b5e1b6.chunk.gperf.d
38bd3ae8bf2bcb7809e6d7eb6bc4a15f875fb9f209dbeb3be7ea4bd32af83558.chunk.kmod.d
39c7f3a84eeb6b6d7fc3af4132130297e029410843f32a48f90d7abb7bed6dcb.chunk.cmake.d
3ece38842b5d21b563b5581db55b8204aff35015b793cee8a58cf19c4bcdd955.chunk.gdbm.d
3ed271afcc1f6a5bb313c0b6326e70c10aac191ab496799230146572c7dce5ad.chunk.usbutils.d
436bd69352994c429f6f7e616c90d54019adc8ab5834532f25930ea6830e0cde.chunk.intltool.d
44724d34b7de7cd2f18e4ff05ddafad4821ce8a9c5eed9275698507f5eec6b51.chunk.xorg-proto-videoproto.d
4561416e873fb0e11ba85d7152afbc7da958d3ab02ca2ba9e532b89a465c4aa0.chunk.mini-utils.d
49fd5e8dd93864ac592bb68611447b10cc6865de1c7ee95cf60833b4254ae30b.chunk.xorg-lib-libXt.d
4e28f0cff9ac1c4f559df7b9157dc7a6df4bcc0bf5000873b3ebf6a5a2344919.chunk.xorg-proto-x11proto.d
516e2befb7929e0159ff906087f8f294013f3a0ddc163840c7a82c67fac17ed9.chunk.perl.d
530077085ceb92e1f3f2a6074e345e3bde2ad8150e822db7b8bb1afbc808108b.chunk.linuxquota.d
530097ee02cdfe16d92b009dab6a28cc0ffb6058194b52fb5f2dc71e506033ea.chunk.systemd.d
5354a927f12cf721fd4e3c2221820f4b58c0cb39a6f7650264addcd90dacf150.chunk.xorg-lib-libXp.d
54bfa662eeed4d863fcb86930b500ac00d87aea41ed6080ac8581f7c087e634e.chunk.googlemock.d
567b1c5ac700e58bb48cd9b6f8f10092c9b0798b55203fe23dc7bfa3017be85b.chunk.gettext.d
56a3503bcfc1684f78b21e6801a1378cef48d5d61fdf73f67ce206182aaaa5eb.chunk.libexpat.d
59c6bd53c5b90089dfbd5bd38ed376e5443b7cca7c132b3125555b3f2af84458.chunk.gawk.d
5a6c63cb34c3780c9a91f907a126d60e52bab4dc817b9855450a75932595d0e8.chunk.xterm.d
5dda75162eea665969b3e25a9c4885f5f89c5ad262cc7e31214d1b584646e487.chunk.DLT-daemon.d
64b11b8a3b51f68006a876585d01781a8e5ea1e0f693427052ae1e931fd56ff7.chunk.xorg-lib-libXtst.d
691f59169e57da19f5da27872e27de8284072b42abe18c089bb9219453952af0.chunk.bzip2.d
692038841cf44ff98333c8ada32cd95f291ce9e2641f09d9d7d4ce81c218cb4b.chunk.freetype2.d
6b1368d1220993637be091cf645c1fc96338b910204d7bbb7b6b5204af292185.chunk.xorg-lib-libXrandr.d
6d08700480c633972ad8fac1937e00211e564c0c83ad82e2b5af6f9e7abe85c4.chunk.xorg-lib-libXfixes.d
6dd098c70d54eca3f910da6ceb62aa56fba0a2dd41ceaced38b46d4d478f7984.chunk.xorg-driver-xf86-video-vesa.d
6e95313133e18cf2a62797d9105edc63cd1e807b537cf14fbdf3f88e292dd873.chunk.xorg-proto-recordproto.d
6ef18864984e6005d7b4846e509327fb3ae4eefa7a8861a8301cd82c8650cdc9.chunk.xorg-lib-libxkbfile.d
70142f1e95c4fa45c75b8f4ac852c2b42e71e00e8e31e38aa7d20da85c252fde.chunk.xorg-font-util.d
704a3076ff0ab0759b2b79f30b1026f5a37191c88abd2a8dfacd2121aaff9f3e.chunk.llvm.d
71907993f2845bd255315ad7540a27d30bb93671e233b5cd9dfa7e19dd8775a5.chunk.ccache.d
7461e12cee1942c0af81bb18b0eca21dacda5fa00c450074eed5b3e6e1e0cf93.chunk.xcb-libxcb.d
76121ebfc97cc82d5feec7e9a2a877ddc22d8c53904f4f7b5ea30606d742aae9.chunk.util-image.d
7755949a4036883926458c02bdab2eeb8bda9b12c7ac3543c882314af0419eff.chunk.xorg-lib-libXScrnSaver.d
78eb9d108e41e27cb96ede555e02a56ed2a75e55656c58a52d45e0d7f6e466ea.chunk.xorg-proto-bigreqsproto.d
7970d957c625958eb20e087d6aea8553e32b2c6fffd4a0d5cd3ca5255004009b.chunk.xorg-driver-xf86-video-cirrus.d
7a85ba8e489b54144bcd43b599719fb4a3de9b4db69e8b536ef02c4d1d0affaf.chunk.freefont-otf.d
7c2662dcb49bfaf5e80d571f6a086d562b1383c51b92cb06dc05ff1d8f5c6f0e.chunk.bash.d
816c2f4f581fa211c6efc2a837fd1988a36da8519da603b7168ed7f14dec9675.chunk.xorg-driver-xf86-input-mouse.d
87c45f052c8d9685e9145a0b48c42fead5a3a5a6d6dd23c19c78bcc49e820054.chunk.xorg-proto-kbproto.d
88aa753fbbcc3f419af5ca2a211a5c1367f9be4da2f3dfc069aee82dcd9837e5.chunk.libpng.d
8de012ed0d7758b1d989158e8e360f681d030e52fc6c0597832e824ff0ea81d8.chunk.autoconf.d
8f08255a4b54a15339d81786d2e979018b90ce910f27cf04427a2c075f142f3e.chunk.xorg-lib-libXpm.d
918448a5de7c40d4f1439061cee2cb02d6a8548ea0219643737c0b7319cb491c.chunk.xorg-proto-xf86vidmodeproto.d
91d257ef0c0ba8ba92ea285a3224dd0533ea83858d56144253cec7c0b075b049.chunk.libxml2.d
950cdd4a4ed9dbf1baa3e63c87667f72897699f122055565c08a23c3819bac9f.chunk.gtk-doc-stub.d
95c2b3e849d4c3e764c304bd88be04dc6841a1ad4588972fd90d4e7cf767300a.chunk.xorg-lib-libxtrans.d
95c826a210847c8d559b1a0e4e7ebcbcc8d0b0782c0b29a813cdb916f54635b3.chunk.dbus-glib.d
967344c84eac90186873b8db6154c1427ba85c70ea60fb487055644bc52fda15.chunk.xorg-lib-libXi.d
99586794def463527f73e438e0cf57762ebc03a358f705bd77ba620d4ccfe8fa.chunk.xcb-pthread-stubs.d
9ad6e3cfef91b1ebaf218ce3c23fc3017ceb13a65b1f4ba96e7fbfb434293a30.chunk.xorg-lib-libfontenc.d
9df834d13aa386cca0e05da72a5faa81780bed6ff22f4ce2a523a2cdca58e95d.chunk.xorg-lib-libSM.d
9e10140ca36a03ec594840662af69ecc17d55ee8d80df7cf2213db095be9509d.chunk.openssh.d
9ecb035bc960f405dc0327cee00ef72f417db1eebe2f525c805893f0fae9402c.chunk.xorg-proto-damageproto.d
9ee1c31536d1bc80c1b326c9b4960c26822999e1845410184b6f00bc23903a54.chunk.xorg-lib-libXrender.d
9fc33ed6bd1fa62200c788f0d543fb7ffab8208b9ae8c4f320410eb16d546889.chunk.openssl.d
a0412bf4198f0353c0915123d1c773fe98e9250adad2d0138b48762981d66240.chunk.util-linux.d
a051639f36af274e0b46d47437e9734059affd006114bfa9f28a625cd3899f70.chunk.xcb-util.d
a24c782b709364dab948f77a6801f961bc265a617936cb3723b39ef1d09c151e.chunk.xorg-lib-libXcomposite.d
a2d0af8326bc15f7c0fc9286847b33dbf3f2901b6983ac022194f954c7e6aeff.chunk.pixman.d
a3fa7808d14091994072ae7d04cc522aab118ea108829ad561eeea27144d53ca.chunk.fontconfig.d
a4aecb4d1026ad65b6f8227f0da363c20397493eff774557cb5352aaf6bec337.chunk.util-wm.d
a506d72173030aedb2308aeb87be1bf3b3e33613e7bb91e819621bc3ad789500.chunk.xorg-util-makedepend.d
a799022847a5f16d36818aeb7b7c870e51ebdae912509b3e4fe67c4dc424aeb8.chunk.xz.d
a87242b4a2cc55c4fa4950df117b731c8c5bf7b42d26cb67146f3bd7749fdc09.chunk.groff.d
a99c3242d1b37443d497f5ae7c23402c111500499e4c63b2ef1f45571d9cfcbf.chunk.xorg-lib-libXRes.d
adb56b770130bfaf94954e6523b71ccc143b6372add8019535df8a8f8e82c4de.chunk.tbdiff.d
adcfb93e29cca334985d14ae40f13c5ca74d4f4a00b70b91a8ea4c9bbb03391a.chunk.attr.d
b05756f16e72054e78da07af5a5f3d4087c91d8a9a0ad668da07b2a747c8446f.chunk.xorg-proto-scrnsaverproto.d
b09d4a0ed2cef5cadb26dd1a4ef5d473f6bde5f010b2bf5c1e15aa0d556ac601.chunk.audiomanager.d
b121fb7601b01edf81cf20a7ede999dbc993f9c2378b3cab6c2639dad9ee5f5b.chunk.xorg-proto-randrproto.d
b1983eaa014dceefad931c68a11ad8ad19f61da334e529355cd8fea14c575488.chunk.xorg-proto-xcmiscproto.d
b3a720e8fee71ac9d72a624be9a4911cc375ad1539f6e9470ea2aff605dc27ac.chunk.bc.d
b408eeea87317fa266192a75f340ab3b7738ed95df8ae855bb707bc9ff653ee3.chunk.gcc.d
b40c6f7c072f71f814f1983efa3380ab44a6d1b6044da59f90fabe51acb4dc5d.chunk.xorg-proto-dmxproto.d
b6f9d66873bc745fafc939ffd39cd330d181399296a45fe8e38dd2c37b4e8396.chunk.xorg-driver-xf86-video-fbdev.d
b718403cea9c15977ed2514573eaff94ee2de9d1448aadafdea23053c537b16f.chunk.libtool.d
b739335b59bef7faba06ac649ff826f07ce0d19687cb9926fa4a74859b7d93c9.chunk.libffi.d
b8928a26eaff606ef64a4e0711a881dbeb117eec57e8024a2c4a0416aaaaf28d.chunk.flex.d
b8d7a95d220abf1b00ea9478637a4b67608a8c79f82b6d54cf3fa6942ea9e434.chunk.xorg-util-macros.d
b96509f3e1a448828f1d0c5461a0d87f8b36835ee4e92d58decdb13e3649c6c6.chunk.xorg-lib-libXext.d
bd113b08aa63d18592c14dc53d21500ac4669ee83db0810711d72f1a09fda421.chunk.drm.d
be677ce893231297cc9d221f10b8f92636c56528f9d94d26a1642b8b0dcf324b.chunk.xorg-lib-libX11.d
be8eb027bcaf165daaf83345067248f442f38ecbd4c16cc11a18fa5022278859.chunk.eglibc.d
bed50bcc0529d6c37ddf7d09e70115a20f4076f1655e9e8c4d56b3d54283e53f.chunk.xorg-lib-libXdamage.d
c0674e73aa2b5abb808c5729762072a53c4e47982318011b323b57d21bcc7d3a.chunk.xorg-lib-libXaw.d
c0c75c8b8cb72428bc63340cf05c1fde4a6f6a41cfa9f54b2bffc5cbe239756a.chunk.xorg-lib-libXfont.d
c236b54002e540b35e5b51504af03cfb2608f861af78c749bdf9909d61b9a750.chunk.sqlite3.d
c29594b7dcacf0cc58b8afb25a2a51d36e42858ccff6401a7453750f2b0fc2a7.chunk.xserver.d
c46fb2c7e047e3bbd9eabd4e7400203620839409d2bb06d8a29b784a23714f7a.chunk.xorg-app-xkbcomp.d
c4b0d77834fd03db6eb6185870f86cac81f1779c77439d6a500815bf36e9161f.chunk.libjpeg.d
c4fa2ade5f1b55782867179a596662c44e3fb83f9e13235359fc7dccc52bd513.chunk.xorg-proto-xineramaproto.d
c57ee2fb791b8de8c985a6a3967d0e2ea88d38a1d3b47200229db229debea423.chunk.xorg-driver-xf86-input-evdev.d
c7ba3e593908efd6bdff8be9e88b290bc5540b8f7c7e61c944b88a85e221e26a.chunk.error-perl-tarball.d
c8a7683b81bcdd42ea37cb0f2a14164198c4f46a05d59f39107c9fb6a501dbed.chunk.lzo.d
ca53a42bc4465c05a3dd868c8d3aa614915f22e2f467d55a221aab052a6aec43.chunk.mesa-x.d
cbe9bee892239945e67aff6fa42325eda8a0c12758bce9abc1311eee1403dd58.chunk.binutils.d
cd656da13be1c8d5719061b40067ae1063751b8ed37d2e3e8d744a3b1d3245a3.chunk.xorg-proto-xf86dgaproto.d
d4c4240815721ea17f837abc7f54df60cebb635a80079b5d42d41b7009e77121.chunk.xorg-proto-glproto.d
d67ec1e18cb82447b5d4dd9e245b94a7387064f9b33840cae43ec0819dc6afff.chunk.xorg-proto-xextproto.d
d7023ffb7a7424f658eb6726e9ebb69a1b6b456ffd9ea9e8e44dc4743895a0c4.chunk.libsha1.d
d97cd2cd0993e3070287cda1be847aa8aa84ba563ffb0ee52af5a8892ed6e904.chunk.git.d
ddb274c19b1d93fc2fb585397eba5f900eafd89631329167310ba81d2f148277.chunk.xorg-proto-inputproto.d
deployments
df5331bd7325ad73b2120f564dd6f64af1152458eb99151fa9f3f933b4471ae4.chunk.btrfs-progs.d
e09bb84be273d3d8a9ea94e25f45429446011f7018cda314329e22241be4cee8.chunk.node-startup-controller.d
e0eb9a82374b59f8e2efa2cc4e3d94de07a273bfea3046d4593bc7f442ecb8bb.chunk.libxslt.d
e111dcb7dac5c3c4150de25b2ca09a42487e8a23c9c86ff376e70994c8211617.chunk.xorg-lib-libXft.d
e3b223e9a9128fb5a1f261196dc9e1cf8298a470df6ba4b534f0966fb772ee4c.chunk.m4.d
e53a6fd82043585ef1399e9f08241ed0571522dcccaba506ff7e90e9b8f3b61a.chunk.pkg-config.d
e5f8c7c3ac3530348d29905c39569151b5dd09d6055d01e833d9507044d81aa5.chunk.googletest.d
e616843d68c150945edae8ec119e10264022e9e556315d9e06decf4f3d6d975b.chunk.xorg-app-xinit.d
e645a2bb47ac55fd16ecc16af9f4f48f524d1ecc6338945fe6f21b669c5a26c8.chunk.xorg-proto-compositeproto.d
e8e0ba99c542f9adea4a604a923b003a76b85344a31f1dc4d877f7e0f4eb4bf3.chunk.glib.d
ecebd8b4b81a06ac03cd30201d39a6c3aa892ef9341f321c3fa8932e49195050.chunk.xorg-proto-fontsproto.d
f03f5197141c649ab1e2cf1a51c0af032500692388815438a60bf9e8a89087a5.chunk.util-keysyms.d
f32e25172bd1cee79ade4f386c816460acd079fccf9121769998e64a826117b3.chunk.xorg-driver-xf86-input-keyboard.d
f67acd61f60e2b988c6bfab4c9af19941dee113880f2223d061e69beea629239.chunk.pciutils.d
f88c2698c239d201741ce70d54836a795017f9350f2536f95b80c7a66ae95dc1.chunk.dbus-pre.d
fd717ebd7da53944fc7b01ab2c5fd8bc332f58278474257aa71650c59b529077.chunk.libcap2.d
staging
tmp1lq8kq
tmpPBKcMu
tmpf8a_TS
tmpiewNgj
tmpkjDEwE
tmpzRd1u5
7:41 a.m.
New subject: Merged: [PATCHv3 0/6] Add morph gc subcommand
On Mon, Jun 24, 2013 at 12:37:10PM +0100, Paul Sherwood wrote:
gc should just *clear* /src/tmp, surely
No, that'd be dangerous. /src/tmp is being used as a general purpose
temporary directory, to replace /tmp and /var/tmp, and deleting files
from there without knowing what the files are, is not acceptable.
Or rather, you, Paul, might only be using it for Morph, but the rest
of us might be using it for other things too.
What we should, in my opinion, be doing is have a dedicated directory
into which Morph puts all of its cached stuff and build artifacts. In
the traditional FHS hierarchy, this would be /var/lib/morph.
--
http://www.codethink.co.uk/ http://wiki.baserock.org/ http://www.baserock.com/
7:53 a.m.
New subject: Merged: [PATCHv3 0/6] Add morph gc subcommand
On 2013-06-24 12:41, Lars Wirzenius wrote:
On Mon, Jun 24, 2013 at 12:37:10PM +0100, Paul Sherwood wrote:
> gc should just *clear* /src/tmp, surely
No, that'd be dangerous. /src/tmp is being used as a general purpose
temporary directory, to replace /tmp and /var/tmp, and deleting files
from there without knowing what the files are, is not acceptable.
Or rather, you, Paul, might only be using it for Morph, but the rest
of us might be using it for other things too.
What we should, in my opinion, be doing is have a dedicated directory
into which Morph puts all of its cached stuff and build artifacts. In
the traditional FHS hierarchy, this would be /var/lib/morph.
i don't think /src/tmp can be standing in for /tmp or /var/tmp?
/src/tmp is specified in the morph.conf file which i created. we
require users to create the /src/tmp directly directory itself
especially for use by the baserock tools, i think? afaik it's not a
required part of a baserock system, but morph requires somewhere for
its
working.
if i store other stuff in a directory called tmp expressly created for
baserock to use, that would seem to be unwise of me.
so i still think morph gc should clear it, unless the change you've
suggested above fixes the problem from a user perspective (ie, gets
rid of the tmp stuff without the user having to clean up manually)
9:12 a.m.
New subject: Merged: [PATCHv3 0/6] Add morph gc subcommand
On Mon, Jun 24, 2013 at 12:53:07PM +0100, Paul Sherwood wrote:
i don't think /src/tmp can be standing in for /tmp or /var/tmp?
/src/tmp is specified in the morph.conf file which i created. we
require users to create the /src/tmp directly directory itself
especially for use by the baserock tools, i think? afaik it's not a
required part of a baserock system, but morph requires somewhere for
its
working.
if i store other stuff in a directory called tmp expressly created for
baserock to use, that would seem to be unwise of me.
You may have created /src/tmp for using with Morph only. I did not. I
created it to replace /tmp (which is uselessly small, and not persistent
across boots) and /var/tmp (which persists, but is uselessly small, being
on the Baserock root disk). And then I set $TMPDIR accordingly. This
is normal Unix practice.
We have been using $TMPDIR (or /tmp, if $TMPDIR is not set, with both
overridden by Morph's --tmpdir option), as the place where Morph keeps
random temporary files, along with other temporary files created by
other entities on the system. At no time has there been an indication
that it's for the sole use of Morph. Again, normal Unix practice.
so i still think morph gc should clear it, unless the change
you've
suggested above fixes the problem from a user perspective (ie, gets
rid of the tmp stuff without the user having to clean up manually)
I will not accept, under any circumstances, a patch to Morph that
intentionally causes data loss to people by deleting files that are
not known to be safe to delete. Having Morph delete everything in
/src/tmp would do that.
It is now clear to me that we need to transition Morph into not using
$TMPDIR, but using a dedicated location instead, with a suitable default
that can be set by the user to something else. With any luck, this can
happen for Baserock 9. After we've done that, there's still old junk
remaining in /src/tmp, and will, unfortunately, have to leave that for
the user to clean up.
--
http://www.codethink.co.uk/ http://wiki.baserock.org/ http://www.baserock.com/
9:23 a.m.
New subject: Merged: [PATCHv3 0/6] Add morph gc subcommand
On 24/06/13 14:12, Lars Wirzenius wrote:
We have been using $TMPDIR (or /tmp, if $TMPDIR is not set, with
both
overridden by Morph's --tmpdir option), as the place where Morph keeps
random temporary files, along with other temporary files created by
other entities on the system. At no time has there been an indication
that it's for the sole use of Morph. Again,
And that is the reason why we split the temporaries used by morph in 4
folders: chunks, deployments, staging and failed. So that we can safely
delete these folders if we want, without affecting other user temporaries.
9:26 a.m.
New subject: Merged: [PATCHv3 0/6] Add morph gc subcommand
On Mon, Jun 24, 2013 at 02:23:02PM +0100, Tiago Gomes wrote:
On 24/06/13 14:12, Lars Wirzenius wrote:
>We have been using $TMPDIR (or /tmp, if $TMPDIR is not set, with both
>overridden by Morph's --tmpdir option), as the place where Morph keeps
>random temporary files, along with other temporary files created by
>other entities on the system. At no time has there been an indication
>that it's for the sole use of Morph. Again,
And that is the reason why we split the temporaries used by morph in
4 folders: chunks, deployments, staging and failed. So that we can
safely delete these folders if we want, without affecting other user
temporaries.
Yes, that is correct. However, keeping those directories in $TMPDIR is
not the right thing to do.
--
http://www.codethink.co.uk/ http://wiki.baserock.org/ http://www.baserock.com/
4:26 a.m.
New subject: Merged: [PATCHv3 0/6] Add morph gc subcommand
On 24/06/2013 14:26, Lars Wirzenius wrote:
> And that is the reason why we split the temporaries used by morph
in
> 4 folders: chunks, deployments, staging and failed. So that we can
> safely delete these folders if we want, without affecting other user
> temporaries.
Yes, that is correct. However, keeping those directories in $TMPDIR is
not the right thing to do.
Revisiting this thread for a status update. Does morph now spot space
overflow situations in time? And does morph gc get rid of all of the
detritus from failed build stagings?
3:56 a.m.
New subject: Merged: [PATCHv3 0/6] Add morph gc subcommand
On Sun, Jul 07, 2013 at 09:26:04AM +0100, Paul Sherwood wrote:
Revisiting this thread for a status update. Does morph now spot
space
overflow situations in time?
I've not had a build fail due to insufficient space since gc has been merged.
It has been over-zealous at times, due to requiring me to clean up
extracted chunks that would actually have been used in the build, but to
know that requires the build graph to be created, at which point you'd
be annoyed that it's failed after so much work has been done.
And does morph gc get rid of all of the detritus from failed build
stagings?
Staging areas for failed builds are moved in all the common build failure
modes, an internal error in morph may leave it behind.
Also, staging areas for deployments are currently not cleaned up, as I had
assumed that this was already taken care of by the deploy plugin.
4:01 a.m.
New subject: Merged: [PATCHv3 0/6] Add morph gc subcommand
On Mon, Jul 08, 2013 at 08:56:41AM +0100, Richard Maw wrote:
Also, staging areas for deployments are currently not cleaned up, as
I had
assumed that this was already taken care of by the deploy plugin.
Is there an RT# so we don't lose track of this?
D.
--
Daniel Silverstone http://www.codethink.co.uk/
Senior Developer GPG 4096/R Key Id: 3CCE BABE 206C 3B69
5:28 a.m.
New subject: Merged: [PATCHv3 0/6] Add morph gc subcommand
On Mon, Jul 08, 2013 at 09:01:39AM +0100, Daniel Silverstone wrote:
On Mon, Jul 08, 2013 at 08:56:41AM +0100, Richard Maw wrote:
> Also, staging areas for deployments are currently not cleaned up, as I had
> assumed that this was already taken care of by the deploy plugin.
Is there an RT# so we don't lose track of this?
There is now, RT#173
4:26 a.m.
New subject: Merged: [PATCHv3 0/6] Add morph gc subcommand
On 08/07/13 08:56, Richard Maw wrote:
Staging areas for failed builds are moved in all the common build
failure modes, an internal error in morph may leave it behind. Also,
staging areas for deployments are currently not cleaned up, as I had
assumed that this was already taken care of by the deploy plugin.
I still believe that we should clean staging areas and deployment caused
by internal errors. As "morph gc" does not support this, I usually
prefer to go to temp dir and do the clean up manually. Also note that a
Ctrl^C during a build or deployment will create garbage that will not be
cleaned by "morph gc". And I don't think that typing Ctrl^C is that
unusually.
Tiago
4:32 a.m.
New subject: Merged: [PATCHv3 0/6] Add morph gc subcommand
On Mon, Jul 08, 2013 at 09:26:50AM +0100, Tiago Gomes wrote:
I still believe that we should clean staging areas and deployment
caused by internal errors.
Me too, I didn't come up with a plan for it because I couldn't think of a
good way to tell if a staging area was currently in use, hence why we have
a separate failed builds directory.
Also note that a Ctrl^C during a build or deployment will create
garbage
that will not be cleaned by "morph gc". And I don't think that
typing Ctrl^C is that unusual.
When I press ^C during a build, the staging area is moved into failed. I
expect it may be timing sensitive for that error.
9:16 a.m.
New subject: Merged: [PATCHv3 0/6] Add morph gc subcommand
On 24/06/13 12:53, Paul Sherwood wrote:
On 2013-06-24 12:41, Lars Wirzenius wrote:
> On Mon, Jun 24, 2013 at 12:37:10PM +0100, Paul Sherwood wrote:
>> gc should just *clear* /src/tmp, surely
>
> No, that'd be dangerous. /src/tmp is being used as a general purpose
> temporary directory, to replace /tmp and /var/tmp, and deleting files
> from there without knowing what the files are, is not acceptable.
>
> Or rather, you, Paul, might only be using it for Morph, but the rest
> of us might be using it for other things too.
>
> What we should, in my opinion, be doing is have a dedicated directory
> into which Morph puts all of its cached stuff and build artifacts. In
> the traditional FHS hierarchy, this would be /var/lib/morph.
i don't think /src/tmp can be standing in for /tmp or /var/tmp?
/src/tmp is specified in the morph.conf file which i created. we
require users to create the /src/tmp directly directory itself
especially for use by the baserock tools, i think? afaik it's not a
required part of a baserock system, but morph requires somewhere for its
working.
if i store other stuff in a directory called tmp expressly created for
baserock to use, that would seem to be unwise of me.
so i still think morph gc should clear it, unless the change you've
suggested above fixes the problem from a user perspective (ie, gets
rid of the tmp stuff without the user having to clean up manually)
If think that the reason for you still have lots of stuff in /src/tmp is
that he chunks that you have in that folder are chunks built before the
new directory structure for temporaries in morph was merged. Now, the
chunks should be installed inside a chunk directory in /src/tmp,
although I am not seeing there a chunk directory.
_______________________________________________
baserock-dev mailing list
baserock-dev(a)baserock.org
http://vlists.pepperfish.net/cgi-bin/mailman/listinfo/baserock-dev-basero...
11:30 a.m.
New subject: [PATCH 5/6] plugins: `morph gc` command via plugin
On 2013-06-07 16:23, Paul Sherwood wrote:
On 07/06/2013 16:17, Richard Maw wrote:
> On Fri, Jun 07, 2013 at 03:58:03PM +0100, Paul Sherwood wrote:
>> On 07/06/2013 15:56, Tiago Gomes wrote:
>>> I think that once a user runs the morph gc command it should
>>> clean everything not necessary, and not stop the cleaning when
>>> there is
>>> X of free space. Besides, not cleaning everything increases the
>>> probability
>>> that the user will have to run morph gc again in a short period of
>>> time.
>>
>> +1
>
> I disagree on the grounds that the aim is to reduce cycle time. Old
> artifacts may still be needed later, and it takes longer to rebuild
> an
> artifact than be notified you don't have enough space and run
> another gc.
>
> I'll change it if people still think everything should be removed,
> but everything has been cached for a reason.
interesting dilemma. yet another setting for morph conf?
It should be an aim that morph generally Does The Right Thing (TM)
rather than needing config..
I think best thing is to remove all items older than x days, and have x
be specifiable on the commandline.
I though that was what Richard and Tiago planned to do =)
Rob
10 a.m.
New subject: [PATCH 6/6] util: suggest how to deal with insufficient space
---
morphlib/util.py | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/morphlib/util.py b/morphlib/util.py
index 6dcb22b..b56443e 100644
--- a/morphlib/util.py
+++ b/morphlib/util.py
@@ -269,4 +269,10 @@ def check_disk_available(tmp_path, tmp_min_size,
'has %(free)d' % locals())
if not errors:
return
- raise morphlib.Error('Insufficient space on disk:\n' +
'\n'.join(errors))
+ raise morphlib.Error('Insufficient space on disk:\n' +
+ '\n'.join(errors) + '\n'
+ 'Please run `morph gc`. If the problem persists '
+ 'increase the disk size, manually clean up some '
+ 'space or reduce the disk space required by the '
+ 'tempdir-min-space and cachedir-min-space '
+ 'configuration options.')
--
1.7.10.4
3616
days inactive
3647
days old
47 comments
6 participants
participants (6)
-
Daniel Silverstone -
Lars Wirzenius -
Paul Sherwood -
Richard Maw -
Rob Taylor -
Tiago Gomes