[gentoo-portage-dev] [PATCH 1/3] Add FEATURES=mount-sandbox to take advantage of mount ns

[Michał Górny]

Support FEATURES=mount-sandbox that unshares the ebuild processes
into a new mount namespace and makes all the mounts private by default.

Signed-off-by: Michał Górny <***@gentoo.org>
---
lib/portage/const.py | 1 +
lib/portage/package/ebuild/doebuild.py | 7 +++++-
lib/portage/process.py | 34 +++++++++++++++++++++-----
man/make.conf.5 | 5 ++++
4 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/lib/portage/const.py b/lib/portage/const.py
index 602caeb34..e0f93f7cc 100644
--- a/lib/portage/const.py
+++ b/lib/portage/const.py
@@ -160,6 +160,7 @@ SUPPORTED_FEATURES = frozenset([
"merge-sync",
"metadata-transfer",
"mirror",
+ "mount-sandbox",
"multilib-strict",
"network-sandbox",
"network-sandbox-proxy",
diff --git a/lib/portage/package/ebuild/doebuild.py b/lib/portage/package/ebuild/doebuild.py
index d0e96f34c..e84a618d2 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -148,6 +148,7 @@ def _doebuild_spawn(phase, settings, actionmap=None, **kwargs):

kwargs['ipc'] = 'ipc-sandbox' not in settings.features or \
phase in _ipc_phases
+ kwargs['mountns'] = 'mount-sandbox' in settings.features
kwargs['networked'] = 'network-sandbox' not in settings.features or \
phase in _networked_phases or \
'network-sandbox' in settings['PORTAGE_RESTRICT'].split()
@@ -1480,7 +1481,8 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi):
# XXX This would be to replace getstatusoutput completely.
# XXX Issue: cannot block execution. Deadlock condition.
def spawn(mystring, mysettings, debug=False, free=False, droppriv=False,
- sesandbox=False, fakeroot=False, networked=True, ipc=True, **keywords):
+ sesandbox=False, fakeroot=False, networked=True, ipc=True,
+ mountns=False, **keywords):
"""
Spawn a subprocess with extra portage-specific options.
Optiosn include:
@@ -1514,6 +1516,8 @@ def spawn(mystring, mysettings, debug=False, free=False, droppriv=False,
@type networked: Boolean
@param ipc: Run this command with host IPC access enabled
@type ipc: Boolean
+ @param mountns: Run this command inside mount namespace
+ @type mountns: Boolean
@param keywords: Extra options encoded as a dict, to be passed to spawn
@type keywords: Dictionary
@rtype: Integer
@@ -1546,6 +1550,7 @@ def spawn(mystring, mysettings, debug=False, free=False, droppriv=False,
if uid == 0 and platform.system() == 'Linux':
keywords['unshare_net'] = not networked
keywords['unshare_ipc'] = not ipc
+ keywords['unshare_mount'] = mountns

if not networked and mysettings.get("EBUILD_PHASE") != "nofetch" and \
("network-sandbox-proxy" in features or "distcc" in features):
diff --git a/lib/portage/process.py b/lib/portage/process.py
index fd326731a..e2ad89b43 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -1,5 +1,5 @@
# portage.py -- core Portage functionality
-# Copyright 1998-2014 Gentoo Foundation
+# Copyright 1998-2018 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2


@@ -10,6 +10,7 @@ import platform
import signal
import socket
import struct
+import subprocess
import sys
import traceback
import os as _os
@@ -222,7 +223,7 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False,
uid=None, gid=None, groups=None, umask=None, logfile=None,
path_lookup=True, pre_exec=None,
close_fds=(sys.version_info < (3, 4)), unshare_net=False,
- unshare_ipc=False, cgroup=None):
+ unshare_ipc=False, unshare_mount=False, cgroup=None):
"""
Spawns a given command.

@@ -260,6 +261,9 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False,
@type unshare_net: Boolean
@param unshare_ipc: If True, IPC will be unshared from the spawned process
@type unshare_ipc: Boolean
+ @param unshare_mount: If True, mount namespace will be unshared and mounts will
+ be private to the namespace
+ @type unshare_mount: Boolean
@param cgroup: CGroup path to bind the process to
@type cgroup: String

@@ -328,7 +332,7 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False,
# This caches the libc library lookup in the current
# process, so that it's only done once rather than
# for each child process.
- if unshare_net or unshare_ipc:
+ if unshare_net or unshare_ipc or unshare_mount:
find_library("c")

# Force instantiation of portage.data.userpriv_groups before the
@@ -344,7 +348,7 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False,
try:
_exec(binary, mycommand, opt_name, fd_pipes,
env, gid, groups, uid, umask, pre_exec, close_fds,
- unshare_net, unshare_ipc, cgroup)
+ unshare_net, unshare_ipc, unshare_mount, cgroup)
except SystemExit:
raise
except Exception as e:
@@ -414,7 +418,7 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False,
return 0

def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
- pre_exec, close_fds, unshare_net, unshare_ipc, cgroup):
+ pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount, cgroup):

"""
Execute a given binary with options
@@ -443,6 +447,9 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
@type unshare_net: Boolean
@param unshare_ipc: If True, IPC will be unshared from the spawned process
@type unshare_ipc: Boolean
+ @param unshare_mount: If True, mount namespace will be unshared and mounts will
+ be private to the namespace
+ @type unshare_mount: Boolean
@param cgroup: CGroup path to bind the process to
@type cgroup: String
@rtype: None
@@ -499,11 +506,13 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
f.write('%d\n' % os.getpid())

# Unshare (while still uid==0)
- if unshare_net or unshare_ipc:
+ if unshare_net or unshare_ipc or unshare_mount:
filename = find_library("c")
if filename is not None:
libc = LoadLibrary(filename)
if libc is not None:
+ # from /usr/include/bits/sched.h
+ CLONE_NEWNS = 0x00020000
CLONE_NEWIPC = 0x08000000
CLONE_NEWNET = 0x40000000

@@ -512,6 +521,9 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
flags |= CLONE_NEWNET
if unshare_ipc:
flags |= CLONE_NEWIPC
+ if unshare_mount:
+ # NEWNS = mount namespace
+ flags |= CLONE_NEWNS

try:
if libc.unshare(flags) != 0:
@@ -519,6 +531,16 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
errno.errorcode.get(ctypes.get_errno(), '?')),
noiselevel=-1)
else:
+ if unshare_mount:
+ # mark the whole filesystem as private to avoid
+ # mounts escaping the namespace
+ s = subprocess.Popen(['mount',
+ '--make-rprivate', '/'])
+ mount_ret = s.wait()
+ if mount_ret != 0:
+ # TODO: should it be fatal maybe?
+ writemsg("Unable to mark mounts private: %d\n" % (mount_ret,),
+ noiselevel=-1)
if unshare_net:
# 'up' the loopback
IFF_UP = 0x1
diff --git a/man/make.conf.5 b/man/make.conf.5
index f69afd015..7cb5741ad 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -494,6 +494,11 @@ ${repository_location}/metadata/md5\-cache/ directory will be used directly
Fetch everything in \fBSRC_URI\fR regardless of \fBUSE\fR settings,
except do not fetch anything when \fImirror\fR is in \fBRESTRICT\fR.
.TP
+.B mount\-sandbox
+Isolate the ebuild phase functions from host mount namespace. This makes
+it possible for ebuild to alter mountpoints without affecting the host
+system. Supported only on Linux. Requires mount namespace support in kernel.
+.TP
.B multilib\-strict
Many Makefiles assume that their libraries should go to /usr/lib, or
$(prefix)/lib. This assumption can cause a serious mess if /usr/lib

[Michał Górny]

Signed-off-by: Michał Górny <***@gentoo.org>
---
bin/pid-ns-init | 25 +++++++++++++++++++++++++
lib/portage/process.py | 11 ++++++-----
2 files changed, 31 insertions(+), 5 deletions(-)
create mode 100644 bin/pid-ns-init

diff --git a/bin/pid-ns-init b/bin/pid-ns-init
new file mode 100644
index 000000000..90660571a
--- /dev/null
+++ b/bin/pid-ns-init
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+# Copyright 2018 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import os
+import sys
+
+
+def main(argv):
+ if len(argv) < 2:
+ return 'Usage: {} <main-child-pid>'.format(argv[0])
+ main_child_pid = int(argv[1])
+
+ # wait for child processes
+ while True:
+ pid, status = os.wait()
+ if pid == main_child_pid:
+ return os.WEXITSTATUS(status)
+
+ # this should never be reached
+ return 127
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv))
diff --git a/lib/portage/process.py b/lib/portage/process.py
index beeb09529..4b0d8f7bc 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -545,13 +545,14 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
else:
if unshare_pid:
# pid namespace requires us to become init
- # TODO: do init-ty stuff
- # therefore, fork() ASAP
fork_ret = os.fork()
if fork_ret != 0:
- pid, status = os.waitpid(fork_ret, 0)
- assert pid == fork_ret
- os._exit(status)
+ os.execv(portage._python_interpreter, [
+ portage._python_interpreter,
+ os.path.join(portage._bin_path,
+ 'pid-ns-init'),
+ '%s' % fork_ret,
+ ])
if unshare_mount:
# mark the whole filesystem as private to avoid
# mounts escaping the namespace

[Michał Górny]

Supporting using PID namespace in order to isolate the ebuild processes
from host system, and make it possible to kill them all easily
(similarly to cgroups but easier to use).

Bug: https://bugs.gentoo.org/659582
Signed-off-by: Michał Górny <***@gentoo.org>
---
lib/portage/const.py | 1 +
lib/portage/package/ebuild/doebuild.py | 8 +++--
lib/portage/process.py | 43 +++++++++++++++++++++++---
man/make.conf.5 | 7 +++++
4 files changed, 52 insertions(+), 7 deletions(-)

diff --git a/lib/portage/const.py b/lib/portage/const.py
index e0f93f7cc..ca66bc46e 100644
--- a/lib/portage/const.py
+++ b/lib/portage/const.py
@@ -174,6 +174,7 @@ SUPPORTED_FEATURES = frozenset([
"notitles",
"parallel-fetch",
"parallel-install",
+ "pid-sandbox",
"prelink-checksums",
"preserve-libs",
"protect-owned",
diff --git a/lib/portage/package/ebuild/doebuild.py b/lib/portage/package/ebuild/doebuild.py
index e84a618d2..9917ac82c 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2018 Gentoo Foundation
+# Copyright 2010-2018 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2

from __future__ import unicode_literals
@@ -152,6 +152,7 @@ def _doebuild_spawn(phase, settings, actionmap=None, **kwargs):
kwargs['networked'] = 'network-sandbox' not in settings.features or \
phase in _networked_phases or \
'network-sandbox' in settings['PORTAGE_RESTRICT'].split()
+ kwargs['pidns'] = 'pid-sandbox' in settings.features

if phase == 'depend':
kwargs['droppriv'] = 'userpriv' in settings.features
@@ -1482,7 +1483,7 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi):
# XXX Issue: cannot block execution. Deadlock condition.
def spawn(mystring, mysettings, debug=False, free=False, droppriv=False,
sesandbox=False, fakeroot=False, networked=True, ipc=True,
- mountns=False, **keywords):
+ mountns=False, pidns=False, **keywords):
"""
Spawn a subprocess with extra portage-specific options.
Optiosn include:
@@ -1518,6 +1519,8 @@ def spawn(mystring, mysettings, debug=False, free=False, droppriv=False,
@type ipc: Boolean
@param mountns: Run this command inside mount namespace
@type mountns: Boolean
+ @param pidns: Run this command in isolated PID namespace
+ @type pidns: Boolean
@param keywords: Extra options encoded as a dict, to be passed to spawn
@type keywords: Dictionary
@rtype: Integer
@@ -1551,6 +1554,7 @@ def spawn(mystring, mysettings, debug=False, free=False, droppriv=False,
keywords['unshare_net'] = not networked
keywords['unshare_ipc'] = not ipc
keywords['unshare_mount'] = mountns
+ keywords['unshare_pid'] = pidns

if not networked and mysettings.get("EBUILD_PHASE") != "nofetch" and \
("network-sandbox-proxy" in features or "distcc" in features):
diff --git a/lib/portage/process.py b/lib/portage/process.py
index e2ad89b43..beeb09529 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -223,7 +223,8 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False,
uid=None, gid=None, groups=None, umask=None, logfile=None,
path_lookup=True, pre_exec=None,
close_fds=(sys.version_info < (3, 4)), unshare_net=False,
- unshare_ipc=False, unshare_mount=False, cgroup=None):
+ unshare_ipc=False, unshare_mount=False, unshare_pid=False,
+ cgroup=None):
"""
Spawns a given command.

@@ -264,6 +265,8 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False,
@param unshare_mount: If True, mount namespace will be unshared and mounts will
be private to the namespace
@type unshare_mount: Boolean
+ @param unshare_pid: If True, PID ns will be unshared from the spawned process
+ @type unshare_pid: Boolean
@param cgroup: CGroup path to bind the process to
@type cgroup: String

@@ -332,7 +335,7 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False,
# This caches the libc library lookup in the current
# process, so that it's only done once rather than
# for each child process.
- if unshare_net or unshare_ipc or unshare_mount:
+ if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
find_library("c")

# Force instantiation of portage.data.userpriv_groups before the
@@ -348,7 +351,8 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False,
try:
_exec(binary, mycommand, opt_name, fd_pipes,
env, gid, groups, uid, umask, pre_exec, close_fds,
- unshare_net, unshare_ipc, unshare_mount, cgroup)
+ unshare_net, unshare_ipc, unshare_mount, unshare_pid,
+ cgroup)
except SystemExit:
raise
except Exception as e:
@@ -418,7 +422,8 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False,
return 0

def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
- pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount, cgroup):
+ pre_exec, close_fds, unshare_net, unshare_ipc, unshare_mount, unshare_pid,
+ cgroup):

"""
Execute a given binary with options
@@ -450,6 +455,8 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
@param unshare_mount: If True, mount namespace will be unshared and mounts will
be private to the namespace
@type unshare_mount: Boolean
+ @param unshare_pid: If True, PID ns will be unshared from the spawned process
+ @type unshare_pid: Boolean
@param cgroup: CGroup path to bind the process to
@type cgroup: String
@rtype: None
@@ -506,7 +513,7 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
f.write('%d\n' % os.getpid())

# Unshare (while still uid==0)
- if unshare_net or unshare_ipc or unshare_mount:
+ if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
filename = find_library("c")
if filename is not None:
libc = LoadLibrary(filename)
@@ -514,6 +521,7 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
# from /usr/include/bits/sched.h
CLONE_NEWNS = 0x00020000
CLONE_NEWIPC = 0x08000000
+ CLONE_NEWPID = 0x20000000
CLONE_NEWNET = 0x40000000

flags = 0
@@ -521,6 +529,10 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
flags |= CLONE_NEWNET
if unshare_ipc:
flags |= CLONE_NEWIPC
+ if unshare_pid:
+ flags |= CLONE_NEWPID
+ # we need to remount /proc, so need mount ns
+ unshare_mount = True
if unshare_mount:
# NEWNS = mount namespace
flags |= CLONE_NEWNS
@@ -531,6 +543,15 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
errno.errorcode.get(ctypes.get_errno(), '?')),
noiselevel=-1)
else:
+ if unshare_pid:
+ # pid namespace requires us to become init
+ # TODO: do init-ty stuff
+ # therefore, fork() ASAP
+ fork_ret = os.fork()
+ if fork_ret != 0:
+ pid, status = os.waitpid(fork_ret, 0)
+ assert pid == fork_ret
+ os._exit(status)
if unshare_mount:
# mark the whole filesystem as private to avoid
# mounts escaping the namespace
@@ -541,6 +562,18 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
# TODO: should it be fatal maybe?
writemsg("Unable to mark mounts private: %d\n" % (mount_ret,),
noiselevel=-1)
+ if unshare_pid:
+ if mount_ret != 0:
+ # can't proceed without private mounts
+ os._exit(1)
+ # mount new /proc for our namespace
+ s = subprocess.Popen(['mount',
+ '-t', 'proc', 'proc', '/proc'])
+ mount_ret = s.wait()
+ if mount_ret != 0:
+ # TODO: fatal?
+ writemsg("Unable to mount new /proc: %d\n" % (mount_ret,),
+ noiselevel=-1)
if unshare_net:
# 'up' the loopback
IFF_UP = 0x1
diff --git a/man/make.conf.5 b/man/make.conf.5
index 7cb5741ad..f98c3a532 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -558,6 +558,13 @@ Use finer\-grained locks when installing packages, allowing for greater
parallelization. For additional parallelization, disable
\fIebuild\-locks\fR.
.TP
+.B pid\-sandbox
+Isolate the process space for the ebuild processes. This makes it
+possible to cleanly kill all processes spawned by the ebuild.
+Supported only on Linux. Requires PID namespace support in kernel.
+
+Implies \fBmount\-sandbox\fR as it requires mounting new /proc.
+.TP
.B prelink\-checksums
If \fBprelink\fR(8) is installed then use it to undo any prelinks on files
before computing checksums for merge and unmerge. This feature is

[Zac Medico]

Post by Michał Górny
@@ -531,6 +543,15 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
errno.errorcode.get(ctypes.get_errno(), '?')),
noiselevel=-1)
+ # pid namespace requires us to become init
+ # TODO: do init-ty stuff
+ # therefore, fork() ASAP
+ fork_ret = os.fork()
+ pid, status = os.waitpid(fork_ret, 0)
+ assert pid == fork_ret
+ os._exit(status)
# mark the whole filesystem as private to avoid
# mounts escaping the namespace
@@ -541,6 +562,18 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
# TODO: should it be fatal maybe?
writemsg("Unable to mark mounts private: %d\n" % (mount_ret,),
noiselevel=-1)
+ # can't proceed without private mounts
+ os._exit(1)

For the benefit of anyone not watching
https://github.com/gentoo/portage/pull/379, the mount_ret is expected
to be non-zero inside a chroot where `mount --make-rprivate /` would
fail because / is not a mountpoint, and this is an extremely valuable
use case for tools like catalyst that perform builds inside a chroot.

Based on /proc/[pid]/mountinfo documentation in the proc(5) manpage,
and empirical analysis of /proc/self/mountinfo in various states,
it should be reasonable to assume that the current propagation flags
are suitable if there is not a shared / or /proc mountpoint found in
/proc/self/mountinfo. We can use a function like this to check if the
`mount --make-rprivate /` call is needed:

def want_make_rprivate():
try:
with open('/proc/self/mountinfo', 'rb') as f:
if re.match(rb'^\S+ \S+ \S+ \S+ (?P<mountpoint>/|/proc) \S+ (?:\S+:\S+ )*(?P<shared>shared:\S+ )', f.read(), re.MULTILINE) is None:
return False
except EnvironmentError:
pass
return True

[Michał Górny]

Post by Zac Medico

Post by Michał Górny
@@ -531,6 +543,15 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
errno.errorcode.get(ctypes.get_errno(), '?')),
noiselevel=-1)
+ # pid namespace requires us to become init
+ # TODO: do init-ty stuff
+ # therefore, fork() ASAP
+ fork_ret = os.fork()
+ pid, status = os.waitpid(fork_ret, 0)
+ assert pid == fork_ret
+ os._exit(status)
# mark the whole filesystem as private to avoid
# mounts escaping the namespace
@@ -541,6 +562,18 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask,
# TODO: should it be fatal maybe?
writemsg("Unable to mark mounts private: %d\n" % (mount_ret,),
noiselevel=-1)
+ # can't proceed without private mounts
+ os._exit(1)

For the benefit of anyone not watching
https://github.com/gentoo/portage/pull/379, the mount_ret is expected
to be non-zero inside a chroot where `mount --make-rprivate /` would
fail because / is not a mountpoint, and this is an extremely valuable
use case for tools like catalyst that perform builds inside a chroot.
Based on /proc/[pid]/mountinfo documentation in the proc(5) manpage,
and empirical analysis of /proc/self/mountinfo in various states,
it should be reasonable to assume that the current propagation flags
are suitable if there is not a shared / or /proc mountpoint found in
/proc/self/mountinfo. We can use a function like this to check if the
return False
pass
return True

Technically speaking, FEATURES=mount-sandbox goes beyond /proc, so
making it guess based on the state of /proc is wrong. Yes, we need only
/proc for the purpose of pid-sandbox but we shouldn't presume the user
wouldn't want more.

Maybe we should split the logic more, and e.g. be more fatal about
--make-rprivate failing when mount-sandbox is explicitly used, and only
care about /proc when pid-sandbox is only used. In the latter case, it
probably doesn't make sense to check but just try to --make-private
/proc. /proc is naturally a mountpoint, so it should cause no harm to
privatize it.

Also, after some extra reading I think we should use 'slave' instead of
'private'. Having things implicitly 'private' might be surprising since
user's actions won't propagate into the build namespace, and I think we
want them to propagate.