Add support for mirroring, showing, and editing remote repos with .udebs. #108

cmd/mirror_create.go

@@ -17,6 +17,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	}
 
 	downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.flags, "with-sources")
+	downloadUdebs := context.flags.Lookup("with-udebs").Value.Get().(bool)
 
 	var (
 		mirrorName, archiveURL, distribution string
@@ -33,7 +34,8 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		archiveURL, distribution, components = args[1], args[2], args[3:]
 	}
 
-	repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(), downloadSources)
+	repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(),
+		downloadSources, downloadUdebs)
 	if err != nil {
 		return fmt.Errorf("unable to create mirror: %s", err)
 	}
@@ -90,6 +92,7 @@ Example:
 
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
+	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
 	cmd.Flag.String("filter", "", "filter packages in mirror")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")

cmd/mirror_edit.go

@@ -27,9 +27,15 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 			repo.FilterWithDeps = flag.Value.Get().(bool)
 		case "with-sources":
 			repo.DownloadSources = flag.Value.Get().(bool)
+		case "with-udebs":
+			repo.DownloadUdebs = flag.Value.Get().(bool)
 		}
 	})
 
+	if repo.IsFlat() && repo.DownloadUdebs {
+		return fmt.Errorf("unable to edit: flat mirrors don't support udebs")
+	}
+
 	if repo.Filter != "" {
 		_, err = query.Parse(repo.Filter)
 		if err != nil {
@@ -74,6 +80,7 @@ Example:
 	cmd.Flag.String("filter", "", "filter packages in mirror")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
+	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
 
 	return cmd
 }

cmd/mirror_show.go

@@ -37,6 +37,11 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 		downloadSources = "yes"
 	}
 	fmt.Printf("Download Sources: %s\n", downloadSources)
+	downloadUdebs := "no"
+	if repo.DownloadUdebs {
+		downloadUdebs = "yes"
+	}
+	fmt.Printf("Download .udebs: %s\n", downloadUdebs)
 	if repo.Filter != "" {
 		fmt.Printf("Filter: %s\n", repo.Filter)
 		filterWithDeps := "no"

system/t04_mirror/CreateMirror25Test_gold

@@ -0,0 +1,4 @@
+Downloading http://mirror.yandex.ru/debian/dists/wheezy/Release...
+
+Mirror [mirror25]: http://mirror.yandex.ru/debian/ wheezy [udeb] successfully added.
+You can run 'aptly mirror update mirror25' to download repository contents.

system/t04_mirror/CreateMirror25Test_mirror_show

@@ -0,0 +1,20 @@
+Name: mirror25
+Archive Root URL: http://mirror.yandex.ru/debian/
+Distribution: wheezy
+Components: main, contrib, non-free
+Architectures: i386
+Download Sources: no
+Download .udebs: yes
+Last update: never
+
+Information from release file:
+Architectures: amd64 armel armhf i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 s390x sparc
+Codename: wheezy
+Components: main contrib non-free
+Date: Sat, 12 Jul 2014 10:59:25 UTC
+Description:  Debian 7.6 Released 12 July 2014
+
+Label: Debian
+Origin: Debian
+Suite: stable
+Version: 7.6

system/t04_mirror/CreateMirror26Test_gold

@@ -0,0 +1 @@
+ERROR: unable to create mirror: debian-installer udebs aren't supported for flat repos

system/t04_mirror/EditMirror8Test_gold

@@ -0,0 +1 @@
+Mirror [wheezy-main]: http://mirror.yandex.ru/debian/ wheezy [udeb] successfully updated.

system/t04_mirror/EditMirror8Test_mirror_show

@@ -0,0 +1,20 @@
+Name: wheezy-main
+Archive Root URL: http://mirror.yandex.ru/debian/
+Distribution: wheezy
+Components: main
+Architectures: i386, amd64
+Download Sources: no
+Download .udebs: yes
+Number of packages: 56121
+
+Information from release file:
+Architectures: amd64 armel armhf i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 s390x sparc
+Codename: wheezy
+Components: main contrib non-free
+Date: Sat, 26 Apr 2014 09:27:11 UTC
+Description:  Debian 7.5 Released 26 April 2014
+
+Label: Debian
+Origin: Debian
+Suite: stable
+Version: 7.5

system/t04_mirror/EditMirror9Test_gold

@@ -0,0 +1 @@
+ERROR: unable to edit: flat mirrors don't support udebs

system/t04_mirror/UpdateMirror12Test_gold

@@ -0,0 +1,33 @@
+
+
+Applying filter...
+Building download queue...
+Download queue: 10 items (0.76 MiB)
+Downloading & parsing package files...
+Downloading http://mirror.yandex.ru/debian/dists/squeeze/InRelease...
+Downloading http://mirror.yandex.ru/debian/dists/squeeze/Release...
+Downloading http://mirror.yandex.ru/debian/dists/squeeze/Release.gpg...
+Downloading http://mirror.yandex.ru/debian/dists/squeeze/main/binary-amd64/Packages.bz2...
+Downloading http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2...
+Downloading http://mirror.yandex.ru/debian/dists/squeeze/main/debian-installer/binary-amd64/Packages.bz2...
+Downloading http://mirror.yandex.ru/debian/dists/squeeze/main/debian-installer/binary-i386/Packages.bz2...
+Downloading http://mirror.yandex.ru/debian/dists/squeeze/non-free/binary-amd64/Packages.bz2...
+Downloading http://mirror.yandex.ru/debian/dists/squeeze/non-free/binary-i386/Packages.bz2...
+Downloading http://mirror.yandex.ru/debian/dists/squeeze/non-free/debian-installer/binary-amd64/Packages.bz2...
+Downloading http://mirror.yandex.ru/debian/dists/squeeze/non-free/debian-installer/binary-i386/Packages.bz2...
+Downloading http://mirror.yandex.ru/debian/pool/main/d/dmraid/dmraid-udeb_1.0.0.rc16-4.1_amd64.udeb...
+Downloading http://mirror.yandex.ru/debian/pool/main/d/dmraid/dmraid-udeb_1.0.0.rc16-4.1_i386.udeb...
+Downloading http://mirror.yandex.ru/debian/pool/main/d/dmraid/dmraid_1.0.0.rc16-4.1_amd64.deb...
+Downloading http://mirror.yandex.ru/debian/pool/main/d/dmraid/dmraid_1.0.0.rc16-4.1_i386.deb...
+Downloading http://mirror.yandex.ru/debian/pool/main/d/dmraid/libdmraid-dev_1.0.0.rc16-4.1_amd64.deb...
+Downloading http://mirror.yandex.ru/debian/pool/main/d/dmraid/libdmraid-dev_1.0.0.rc16-4.1_i386.deb...
+Downloading http://mirror.yandex.ru/debian/pool/main/d/dmraid/libdmraid1.0.0.rc16-udeb_1.0.0.rc16-4.1_amd64.udeb...
+Downloading http://mirror.yandex.ru/debian/pool/main/d/dmraid/libdmraid1.0.0.rc16-udeb_1.0.0.rc16-4.1_i386.udeb...
+Downloading http://mirror.yandex.ru/debian/pool/main/d/dmraid/libdmraid1.0.0.rc16_1.0.0.rc16-4.1_amd64.deb...
+Downloading http://mirror.yandex.ru/debian/pool/main/d/dmraid/libdmraid1.0.0.rc16_1.0.0.rc16-4.1_i386.deb...
+Mirror `squeeze` has been successfully updated.
+Packages filtered: 45830 -> 10.
+gpgv: Good signature from "Debian Archive Automatic Signing Key (6.0/squeeze) <ftpmaster@debian.org>"
+gpgv: Good signature from "Squeeze Stable Release Key <debian-release@lists.debian.org>"
+gpgv:  RSA key ID 473041FA
+gpgv:  RSA key ID B98321F9

system/t04_mirror/create.py

@@ -277,7 +277,7 @@ class CreateMirror23Test(BaseTest):
 
 class CreateMirror24Test(BaseTest):
     """
-    create mirror: mirror with wrong filter
+    create mirror: disable config value with option
     """
     runCmd = "aptly mirror create -ignore-signatures=false -keyring=aptlytest.gpg mirror24 http://security.debian.org/ wheezy/updates main"
     fixtureGpg = True
@@ -286,3 +286,23 @@ class CreateMirror24Test(BaseTest):
     configOverride = {
         "gpgDisableVerify": True
     }
+
+
+class CreateMirror25Test(BaseTest):
+    """
+    create mirror: mirror with udebs enabled
+    """
+    runCmd = "aptly -architectures=i386 mirror create -ignore-signatures -with-udebs mirror25 http://mirror.yandex.ru/debian/ wheezy"
+
+    def check(self):
+        self.check_output()
+        self.check_cmd_output("aptly mirror show mirror25", "mirror_show")
+
+
+class CreateMirror26Test(BaseTest):
+    """
+    create mirror: flat mirror with udebs
+    """
+    runCmd = "aptly mirror create -keyring=aptlytest.gpg -with-udebs mirror26 http://pkg.jenkins-ci.org/debian-stable binary/"
+    fixtureGpg = True
+    expectedCode = 1

system/t04_mirror/edit.py

@@ -79,3 +79,25 @@ class EditMirror7Test(BaseTest):
     fixtureDB = True
     runCmd = "aptly mirror edit -architectures=amd64,x56 wheezy-main"
     expectedCode = 1
+
+
+class EditMirror8Test(BaseTest):
+    """
+    edit mirror: enable udebs
+    """
+    fixtureDB = True
+    runCmd = "aptly mirror edit -with-udebs wheezy-main"
+
+    def check(self):
+        self.check_output()
+        self.check_cmd_output("aptly mirror show wheezy-main", "mirror_show", match_prepare=lambda s: re.sub(r"Last update: [0-9:+A-Za-z -]+\n", "", s))
+
+
+class EditMirror9Test(BaseTest):
+    """
+    edit mirror: flat mirror with udebs
+    """
+    fixtureCmds = ["aptly mirror create -keyring=aptlytest.gpg mirror9 http://pkg.jenkins-ci.org/debian-stable binary/"]
+    fixtureGpg = True
+    runCmd = "aptly mirror edit -with-udebs mirror9"
+    expectedCode = 1

system/t04_mirror/update.py

@@ -156,3 +156,19 @@ class UpdateMirror11Test(BaseTest):
 
     def output_processor(self, output):
         return "\n".join(sorted(output.split("\n")))
+
+
+class UpdateMirror12Test(BaseTest):
+    """
+    update mirrors: update with udebs
+    """
+    longTest = False
+    fixtureGpg = True
+    fixtureCmds = [
+        "aptly -architectures=i386,amd64 mirror create -keyring=aptlytest.gpg -filter='$$Source (dmraid)' -with-udebs squeeze http://mirror.yandex.ru/debian/ squeeze main non-free",
+    ]
+    runCmd = "aptly mirror update -keyring=aptlytest.gpg squeeze"
+    outputMatchPrepare = lambda _, s: re.sub(r'Signature made .* using', '', s)
+
+    def output_processor(self, output):
+        return "\n".join(sorted(output.split("\n")))