New upstream version 1.5.0+ds1

system/lib.py

@@ -8,21 +8,32 @@ import json
 import subprocess
 import os
 import posixpath
+import re
 import shlex
 import shutil
 import string
 import threading
-import urllib
+import urllib.error
+import urllib.parse
+import urllib.request
 import pprint
-import SocketServer
-import SimpleHTTPServer
+import socketserver
+import http.server
+import zlib
 
 
-class ThreadedTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
+def ungzip_if_required(output):
+    if isinstance(output, bytes) and output.startswith(b"\x1f\x8b"):
+        return zlib.decompress(output, 16 + zlib.MAX_WBITS).decode('utf-8')
+
+    return output
+
+
+class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
     pass
 
 
-class FileHTTPServerRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
+class FileHTTPServerRequestHandler(http.server.SimpleHTTPRequestHandler):
     def translate_path(self, path):
         """Translate a /-separated PATH to the local filename syntax.
 
@@ -34,13 +45,13 @@ class FileHTTPServerRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
         # abandon query parameters
         path = path.split('?', 1)[0]
         path = path.split('#', 1)[0]
-        path = posixpath.normpath(urllib.unquote(path))
+        path = posixpath.normpath(urllib.parse.unquote(path))
         words = path.split('/')
-        words = filter(None, words)
+        words = [_f for _f in words if _f]
         path = self.rootPath
         for word in words:
-            drive, word = os.path.splitdrive(word)
-            head, word = os.path.split(word)
+            _, word = os.path.splitdrive(word)
+            _, word = os.path.split(word)
             if word in (os.curdir, os.pardir):
                 continue
             path = os.path.join(path, word)
@@ -69,7 +80,7 @@ class GPGFinder(object):
     def find_gpg(self, executables, expected_version):
         for executable in executables:
             try:
-                output = subprocess.check_output([executable, "--version"])
+                output = subprocess.check_output([executable, "--version"], text=True)
                 if expected_version in output:
                     return executable
             except Exception:
@@ -78,11 +89,31 @@ class GPGFinder(object):
         return None
 
 
+class DotFinder(object):
+    """
+    dot binary discovery.
+    """
+
+    def __init__(self):
+        self.dot = self.find_dot(["dot"])
+
+    def find_dot(self, executables):
+        for executable in executables:
+            try:
+                subprocess.check_output([executable, "-V"], text=True)
+                return executable
+            except Exception:
+                pass
+
+        return None
+
+
 class BaseTest(object):
     """
     Base class for all tests.
     """
 
+    skipTest = False
     longTest = False
     fixturePool = False
     fixturePoolCopy = False
@@ -92,12 +123,16 @@ class BaseTest(object):
     requiresFTP = False
     requiresGPG1 = False
     requiresGPG2 = False
+    requiresDot = False
+    sortOutput = False
 
     expectedCode = 0
     configFile = {
         "rootDir": "%s/.aptly" % os.environ["HOME"],
         "downloadConcurrency": 4,
         "downloadSpeedLimit": 0,
+        "downloadRetries": 5,
+        "databaseOpenAttempts": 10,
         "architectures": [],
         "dependencyFollowSuggests": False,
         "dependencyFollowRecommends": False,
@@ -107,6 +142,7 @@ class BaseTest(object):
         "gpgDisableSign": False,
         "ppaDistributorID": "ubuntu",
         "ppaCodename": "",
+        "enableMetricsEndpoint": True,
     }
     configOverride = {}
     environmentOverride = {}
@@ -126,11 +162,15 @@ class BaseTest(object):
     captureResults = False
 
     gpgFinder = GPGFinder()
+    dotFinder = DotFinder()
 
     def test(self):
         self.prepare()
-        self.run()
-        self.check()
+        try:
+            self.run()
+            self.check()
+        finally:
+            self.teardown()
 
     def prepare_remove_all(self):
         if os.path.exists(os.path.join(os.environ["HOME"], ".aptly")):
@@ -138,7 +178,8 @@ class BaseTest(object):
         if os.path.exists(os.path.join(os.environ["HOME"], ".aptly.conf")):
             os.remove(os.path.join(os.environ["HOME"], ".aptly.conf"))
         if os.path.exists(os.path.join(os.environ["HOME"], ".gnupg", "aptlytest.gpg")):
-            os.remove(os.path.join(os.environ["HOME"], ".gnupg", "aptlytest.gpg"))
+            os.remove(os.path.join(
+                os.environ["HOME"], ".gnupg", "aptlytest.gpg"))
 
     def prepare_default_config(self):
         cfg = self.configFile.copy()
@@ -162,24 +203,29 @@ class BaseTest(object):
             return False
         if self.requiresGPG2 and self.gpgFinder.gpg2 is None:
             return False
+        if self.requiresDot and self.dotFinder.dot is None:
+            return False
 
         return True
 
     def prepare_fixture(self):
         if self.fixturePool:
-            os.makedirs(os.path.join(os.environ["HOME"], ".aptly"), 0755)
-            os.symlink(self.fixturePoolDir, os.path.join(os.environ["HOME"], ".aptly", "pool"))
+            os.makedirs(os.path.join(os.environ["HOME"], ".aptly"), 0o755)
+            os.symlink(self.fixturePoolDir, os.path.join(
+                os.environ["HOME"], ".aptly", "pool"))
 
         if self.fixturePoolCopy:
-            os.makedirs(os.path.join(os.environ["HOME"], ".aptly"), 0755)
-            shutil.copytree(self.fixturePoolDir, os.path.join(os.environ["HOME"], ".aptly", "pool"), ignore=shutil.ignore_patterns(".git"))
+            os.makedirs(os.path.join(os.environ["HOME"], ".aptly"), 0o755)
+            shutil.copytree(self.fixturePoolDir, os.path.join(
+                os.environ["HOME"], ".aptly", "pool"), ignore=shutil.ignore_patterns(".git"))
 
         if self.fixtureDB:
-            shutil.copytree(self.fixtureDBDir, os.path.join(os.environ["HOME"], ".aptly", "db"))
+            shutil.copytree(self.fixtureDBDir, os.path.join(
+                os.environ["HOME"], ".aptly", "db"))
 
         if self.fixtureWebServer:
             self.webServerUrl = self.start_webserver(os.path.join(os.path.dirname(inspect.getsourcefile(self.__class__)),
-                                                     self.fixtureWebServer))
+                                                                  self.fixtureWebServer))
 
         if self.requiresGPG2:
             self.run_cmd([
@@ -194,11 +240,17 @@ class BaseTest(object):
             for cmd in self.fixtureCmds:
                 self.run_cmd(cmd)
 
+    def sort_lines(self, output):
+        return "\n".join(sorted(self.ensure_utf8(output).split("\n")))
+
     def run(self):
-        self.output = self.output_processor(self.run_cmd(self.runCmd, self.expectedCode))
+        output = self.run_cmd(self.runCmd, self.expectedCode)
+        if self.sortOutput:
+            output = self.sort_lines(output)
+        self.output = self.output_processor(output)
 
     def _start_process(self, command, stderr=subprocess.STDOUT, stdout=None):
-        if not hasattr(command, "__iter__"):
+        if isinstance(command, str):
             params = {
                 'files': os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files"),
                 'changes': os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "changes"),
@@ -210,8 +262,8 @@ class BaseTest(object):
                 params['url'] = self.webServerUrl
 
             command = string.Template(command).substitute(params)
-
             command = shlex.split(command)
+
         environ = os.environ.copy()
         environ["LC_ALL"] = "C"
         environ.update(self.environmentOverride)
@@ -223,10 +275,12 @@ class BaseTest(object):
             output, _ = proc.communicate()
             if expected_code is not None:
                 if proc.returncode != expected_code:
-                    raise Exception("exit code %d != %d (output: %s)" % (proc.returncode, expected_code, output))
+                    raise Exception("exit code %d != %d (output: %s)" % (
+                        proc.returncode, expected_code, output))
             return output
-        except Exception, e:
-            raise Exception("Running command %s failed: %s" % (command, str(e)))
+        except Exception as e:
+            raise Exception("Running command %s failed: %s" %
+                            (command, str(e)))
 
     def gold_processor(self, gold):
         return gold
@@ -243,9 +297,19 @@ class BaseTest(object):
     def get_gold(self, gold_name="gold"):
         return self.gold_processor(open(self.get_gold_filename(gold_name), "r").read())
 
+    def strip_retry_lines(self, s):
+        for prefix in (
+                'Following redirect',
+                'Error downloading',
+                'Retrying',
+        ):
+            s = re.sub(r'{}.*\n'.format(prefix), '', s)
+        return s
+
     def check_output(self):
         try:
-            self.verify_match(self.get_gold(), self.output, match_prepare=self.outputMatchPrepare)
+            self.verify_match(self.get_gold(), self.output,
+                              match_prepare=self.outputMatchPrepare)
         except:  # noqa: E722
             if self.captureResults:
                 if self.outputMatchPrepare is not None:
@@ -268,18 +332,20 @@ class BaseTest(object):
             else:
                 raise
 
-    def read_file(self, path):
-        with open(os.path.join(os.environ["HOME"], ".aptly", path), "r") as f:
+    def read_file(self, path, mode=''):
+        with open(os.path.join(os.environ["HOME"], ".aptly", path), "r" + mode) as f:
             return f.read()
 
     def delete_file(self, path):
         os.unlink(os.path.join(os.environ["HOME"], ".aptly", path))
 
-    def check_file_contents(self, path, gold_name, match_prepare=None):
-        contents = self.read_file(path)
+    def check_file_contents(self, path, gold_name, match_prepare=None, mode='', ensure_utf8=True):
+        contents = self.read_file(path, mode=mode)
         try:
 
-            self.verify_match(self.get_gold(gold_name), contents, match_prepare=match_prepare)
+            self.verify_match(self.get_gold(gold_name),
+                              contents, match_prepare=match_prepare,
+                              ensure_utf8=ensure_utf8)
         except:  # noqa: E722
             if self.captureResults:
                 if match_prepare is not None:
@@ -328,23 +394,38 @@ class BaseTest(object):
         if item not in l:
             raise Exception("item %r not in %r", item, l)
 
+    def check_not_in(self, item, l):
+        if item in l:
+            raise Exception("item %r in %r", item, l)
+
     def check_subset(self, a, b):
         diff = ''
-        for k, v in a.items():
+        for k, v in list(a.items()):
             if k not in b:
                 diff += "unexpected key '%s'\n" % (k,)
             elif b[k] != v:
-                diff += "wrong value '%s' for key '%s', expected '%s'\n" % (v, k, b[k])
+                diff += "wrong value '%s' for key '%s', expected '%s'\n" % (
+                    v, k, b[k])
         if diff:
             raise Exception("content doesn't match:\n" + diff)
 
-    def verify_match(self, a, b, match_prepare=None):
+    def ensure_utf8(self, a):
+        if isinstance(a, bytes):
+            return a.decode('utf-8')
+        return a
+
+    def verify_match(self, a, b, match_prepare=None, ensure_utf8=True):
+        if ensure_utf8:
+            a = self.ensure_utf8(a)
+            b = self.ensure_utf8(b)
+
         if match_prepare is not None:
             a = match_prepare(a)
             b = match_prepare(b)
 
         if a != b:
-            diff = "".join(difflib.unified_diff([l + "\n" for l in a.split("\n")], [l + "\n" for l in b.split("\n")]))
+            diff = "".join(difflib.unified_diff(
+                [l + "\n" for l in a.split("\n")], [l + "\n" for l in b.split("\n")]))
 
             raise Exception("content doesn't match:\n" + diff + "\n")
 
@@ -355,9 +436,13 @@ class BaseTest(object):
         self.prepare_default_config()
         self.prepare_fixture()
 
+    def teardown(self):
+        pass
+
     def start_webserver(self, directory):
         FileHTTPServerRequestHandler.rootPath = directory
-        self.webserver = ThreadedTCPServer(("localhost", 0), FileHTTPServerRequestHandler)
+        self.webserver = ThreadedTCPServer(
+            ("localhost", 0), FileHTTPServerRequestHandler)
 
         server_thread = threading.Thread(target=self.webserver.serve_forever)
         server_thread.daemon = True