[gentoo-portage-dev] [PATCH 1/2] SyncBase: split out _refresh_keys method (bug 660732)

[Zac Medico]

Split out a _refresh_keys method from the RsyncSync class,
so GitSync can use it for retry support.

Bug: https://bugs.gentoo.org/660732
---
pym/portage/sync/modules/rsync/rsync.py | 33 +-------------------------
pym/portage/sync/syncbase.py | 42 +++++++++++++++++++++++++++++++++
2 files changed, 43 insertions(+), 32 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 382a1eaaef..a715e2818b 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -7,7 +7,6 @@ import time
import signal
import socket
import datetime
-import functools
import io
import re
import random
@@ -23,10 +22,8 @@ good = create_color_func("GOOD")
bad = create_color_func("BAD")
warn = create_color_func("WARN")
from portage.const import VCS_DIRS, TIMESTAMP_FORMAT, RSYNC_PACKAGE_ATOM
-from portage.util._eventloop.global_event_loop import global_event_loop
from portage.util import writemsg, writemsg_stdout
from portage.util.futures import asyncio
-from portage.util.futures.executor.fork import ForkExecutor
from portage.sync.getaddrinfo_validate import getaddrinfo_validate
from _emerge.UserQuery import UserQuery
from portage.sync.syncbase import NewBase
@@ -149,39 +146,11 @@ class RsyncSync(NewBase):
# will not be performed and the user will have to fix it and try again,
# so we may as well bail out before actual rsync happens.
if openpgp_env is not None and self.repo.sync_openpgp_key_path is not None:
-
try:
out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
openpgp_env.import_key(f)
- out.ebegin('Refreshing keys from keyserver')
- retry_decorator = self._key_refresh_retry_decorator()
- if retry_decorator is None:
- openpgp_env.refresh_keys()
- else:
- def noisy_refresh_keys():
- """
- Since retry does not help for some types of
- errors, display errors as soon as they occur.
- """
- try:
- openpgp_env.refresh_keys()
- except Exception as e:
- writemsg_level("%s\n" % (e,),
- level=logging.ERROR, noiselevel=-1)
- raise # retry
-
- # The ThreadPoolExecutor that asyncio uses by default
- # does not support cancellation of tasks, therefore
- # use ForkExecutor for task cancellation support, in
- # order to enforce timeouts.
- loop = global_event_loop()
- with ForkExecutor(loop=loop) as executor:
- func_coroutine = functools.partial(loop.run_in_executor,
- executor, noisy_refresh_keys)
- decorated_func = retry_decorator(func_coroutine, loop=loop)
- loop.run_until_complete(decorated_func())
- out.eend(0)
+ self._refresh_keys(openpgp_env)
except (GematoException, asyncio.TimeoutError) as e:
writemsg_level("!!! Manifest verification impossible due to keyring problem:\n%s\n"
% (e,),
diff --git a/pym/portage/sync/syncbase.py b/pym/portage/sync/syncbase.py
index 7d4d33272e..fe13f16434 100644
--- a/pym/portage/sync/syncbase.py
+++ b/pym/portage/sync/syncbase.py
@@ -7,13 +7,17 @@ This class contains common initialization code and functions.
'''

from __future__ import unicode_literals
+import functools
+import io
import logging
import os

import portage
from portage.util import writemsg_level
+from portage.util._eventloop.global_event_loop import global_event_loop
from portage.util.backoff import RandomExponentialBackoff
from portage.util.futures.retry import retry
+from portage.util.futures.executor.fork import ForkExecutor
from . import _SUBMODULE_PATH_MAP

class SyncBase(object):
@@ -189,6 +193,44 @@ class SyncBase(object):
multiplier=(1 if retry_delay_mult is None else retry_delay_mult),
base=(2 if retry_delay_exp_base is None else retry_delay_exp_base)))

+ def _refresh_keys(self, openpgp_env):
+ """
+ Refresh keys stored in openpgp_env. Raises gemato.exceptions.GematoException
+ or asyncio.TimeoutError on failure.
+
+ @param openpgp_env: openpgp environment
+ @type openpgp_env: gemato.openpgp.OpenPGPEnvironment
+ """
+ out = portage.output.EOutput(quiet=('--quiet' in self.options['emerge_config'].opts))
+ out.ebegin('Refreshing keys from keyserver')
+ retry_decorator = self._key_refresh_retry_decorator()
+ if retry_decorator is None:
+ openpgp_env.refresh_keys()
+ else:
+ def noisy_refresh_keys():
+ """
+ Since retry does not help for some types of
+ errors, display errors as soon as they occur.
+ """
+ try:
+ openpgp_env.refresh_keys()
+ except Exception as e:
+ writemsg_level("%s\n" % (e,),
+ level=logging.ERROR, noiselevel=-1)
+ raise # retry
+
+ # The ThreadPoolExecutor that asyncio uses by default
+ # does not support cancellation of tasks, therefore
+ # use ForkExecutor for task cancellation support, in
+ # order to enforce timeouts.
+ loop = global_event_loop()
+ with ForkExecutor(loop=loop) as executor:
+ func_coroutine = functools.partial(loop.run_in_executor,
+ executor, noisy_refresh_keys)
+ decorated_func = retry_decorator(func_coroutine, loop=loop)
+ loop.run_until_complete(decorated_func())
+ out.eend(0)
+

class NewBase(SyncBase):
'''Subclasses Syncbase adding a new() and runs it

[Zac Medico]

Bug: https://bugs.gentoo.org/660732
---
pym/portage/sync/modules/git/git.py | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/modules/git/git.py b/pym/portage/sync/modules/git/git.py
index 97c4c1de67..68f8bd1fb9 100644
--- a/pym/portage/sync/modules/git/git.py
+++ b/pym/portage/sync/modules/git/git.py
@@ -8,6 +8,7 @@ import subprocess
import portage
from portage import os
from portage.util import writemsg_level, shlex_split
+from portage.util.futures import asyncio
from portage.output import create_color_func, EOutput
good = create_color_func("GOOD")
bad = create_color_func("BAD")
@@ -197,10 +198,8 @@ class GitSync(NewBase):
out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
openpgp_env.import_key(f)
- out.ebegin('Refreshing keys from keyserver')
- openpgp_env.refresh_keys()
- out.eend(0)
- except GematoException as e:
+ self._refresh_keys(openpgp_env)
+ except (GematoException, asyncio.TimeoutError) as e:
writemsg_level("!!! Verification impossible due to keyring problem:\n%s\n"
% (e,),
level=logging.ERROR, noiselevel=-1)