Merge pull request #36 from chutzimir/support-insecure-repos

Support insecure repos

1 files changed, 200 insertions, 75 deletions

diff --git a/apt-mirror b/apt-mirror
index 0c52b0b..9d2bcbf 100755
--- a/apt-mirror
+++ b/apt-mirror
@@ -125,12 +125,18 @@ my %config_variables = (
 my @config_binaries = ();
 my @config_sources  = ();
 
+my @release_urls;
 my @index_urls;
 my @childrens       = ();
 my %skipclean       = ();
 my %clean_directory = ();
+my @hash_strength   = qw(SHA512 SHA256 SHA1 MD5Sum);
 
-my %sha256_filenames = ();
+# Mapping of files downloaded from a by-hash directory to their canonical locations.
+my %hashsum_to_files = ();
+
+# Mapping of all the checksums for a given canonical filename.
+my %file_to_hashsums;
 
 ######################################################################################
 ## Setting up $config_file variable
@@ -296,17 +302,15 @@ sub download_urls
     }
     print "\nEnd time: " . localtime() . "\n\n";
 
-    if (scalar keys %sha256_filenames > 0)
+    if (scalar keys %hashsum_to_files > 0)
     {
-        print "Begin linking checksums to filenames...\n";
-        foreach my $hash_filename (keys %sha256_filenames)
+        foreach my $hashsum_filename (keys %hashsum_to_files)
         {
-            foreach my $filename (@{$sha256_filenames{$hash_filename}})
+            foreach my $filename (@{$hashsum_to_files{$hashsum_filename}})
             {
-                copy_file( $hash_filename, $filename );
+                copy_file( $hashsum_filename, $filename );
             }
         }
-        print "End linking checksums to filenames...\n";
     }
 
 }
@@ -315,7 +319,7 @@ sub download_urls
 
 sub parse_config_line
 {
-    my $pattern_deb_line = qr/^[\t ]*(?<type>deb-src|deb)(?:-(?<arch>[\w\-]+))?[\t ]+(?:\[(?<options>[^\]]+)\][\t ]+)?(?<uri>[^\s]+)[\t ]+(?<components>.*)$/;
+    my $pattern_deb_line = qr/^[\t ]*(?<type>deb-src|deb)(?:-(?<arch>[\w\-]+))?[\t ]+(?:\[(?<options>[^\]]+)\][\t ]+)?(?<uri>[^\s]+)[\t ]+(?<components>.+)$/;
     my $line = $_;
     my %config;
     if ( $line =~ $pattern_deb_line ) {
@@ -424,24 +428,42 @@ sub add_url_to_download
 {
     my $url = remove_double_slashes(shift);
     my $size = shift;
-    my $sha256 = shift;
+    my $strongest_hash = shift;
+    my $hash = shift;
+    my $hashsum = shift;
 
     my $canonical_filename = sanitise_uri($url);
     $skipclean{$canonical_filename} = 1;
 
-    if ($sha256)
+    if ($hashsum)
     {
-        # Download from the "by-hash" directory, and make a copy (or link) it
-        # in the canonical location
-        $url = dirname($url) . "/by-hash/SHA256/" . $sha256;
+        # If the optional hashsum was passed as an argument
+        # - download the strongest hash only
+        # - make a copy to the canonical location
+        # - make a copy for the other known hash versions
+
+        $url = dirname($url) . "/by-hash/${hash}/${hashsum}";
 
-        my $hash_filename = dirname($canonical_filename) . "/by-hash/SHA256/" . $sha256;
+        my $hashsum_filename = dirname($canonical_filename) . "/by-hash/${hash}/${hashsum}";
+        $skipclean{$hashsum_filename} = 1;
 
-        $sha256_filenames{$hash_filename} ||= [];
-        push @{$sha256_filenames{$hash_filename}}, $canonical_filename;
-        $skipclean{$hash_filename} = 1;
+        if ($hash eq $strongest_hash)
+        {
+            # This is the strongest hash, which is the one to download.
+            # Also need to remember to which canonical location it should be linked.
+            $hashsum_to_files{$hashsum_filename} ||= [];
+            push @{$hashsum_to_files{$hashsum_filename}}, $canonical_filename;
+            $urls_to_download{$url} = $size;
+        } else {
+            # We are not going to download using this checksum, but we still
+            # need to know where to put the checksum.
+            $file_to_hashsums{$canonical_filename} ||= [];
+            push @{$file_to_hashsums{$canonical_filename}}, $hashsum_filename;
+        }
+    } else {
+        # Not using by-hash, so download the file only.
+        $urls_to_download{$url} = $size;
     }
-    $urls_to_download{$url} = $size;
 }
 
 foreach (@config_sources)
@@ -454,9 +476,7 @@ foreach (@config_sources)
     }
     else
     {
-        # https://wiki.debian.org/DebianRepository/Format#Flat_Repository_Format
         $url = $uri . "/" . $distribution . "/";
-        add_url_to_download( $url . "Sources" );
     }
 
     add_url_to_download( $url . "InRelease" );
@@ -475,9 +495,7 @@ foreach (@config_binaries)
     }
     else
     {
-        # https://wiki.debian.org/DebianRepository/Format#Flat_Repository_Format
         $url = $uri . "/" . $distribution . "/";
-        add_url_to_download( $url . "Packages" );
     }
 
     add_url_to_download( $url . "InRelease" );
@@ -487,8 +505,8 @@ foreach (@config_binaries)
 }
 
 chdir get_variable("skel_path") or die("apt-mirror: can't chdir to skel");
-@index_urls = sort keys %urls_to_download;
-download_urls( "index", @index_urls );
+@release_urls = sort keys %urls_to_download;
+download_urls( "release", @release_urls );
 
 ######################################################################################
 ## Download all relevant metadata
@@ -505,6 +523,7 @@ sub find_metadata_in_release
     my $arch_regex = "(?:${arch}|all)";
     my $compressed_extension_regex = '(?:\.(?:gz|bz2|xz|lzma))$';
     my $dist_uri;
+    my $hash_type_regex = "(?:" . join("|", @hash_strength) . ")";
 
     if (@components)
     {
@@ -514,73 +533,96 @@ sub find_metadata_in_release
     else {
         $dist_uri  = remove_double_slashes($uri . "/" . $distribution . "/");
     }
-    $release_uri  = $dist_uri . "Release";
-    $release_path = get_variable("skel_path") . "/" . sanitise_uri($release_uri);
 
-    unless ( open STREAM, "<$release_path" )
+    my $stream;
+    foreach my $release_filename ("InRelease", "Release")
     {
-        warn( "Failed to open Release file from " . $release_uri );
-        return;
+        $release_uri  = $dist_uri . $release_filename;
+        $release_path = get_variable("skel_path") . "/" . sanitise_uri($release_uri);
+
+        last if ( open $stream, "<", $release_path);
+        $stream = undef;
     }
 
-    my $checksums = 0;
+    unless ( $stream )
+    {
+        warn( "Failed to find InRelease or Release in " . get_variable("skel_path") . "/" . sanitise_uri($dist_uri) );
+        return 0;
+    }
+
+
+    my $hash = undef;
+    my %avaiable_hashes = ();
     my $acquire_by_hash = 0;
     my @parts_to_download = ();
-    while ( $line = <STREAM> )
+    while ( $line = <$stream> )
     {
         chomp $line;
-        if ($checksums)
+        if ($hash)
         {
             if ( $line =~ /^ +(.*)$/ )
             {
                 my @parts = split( / +/, $1 );
                 if ( @parts == 3 )
                 {
-                    my ( $sha256, $size, $filename ) = @parts;
+                    my ( $hashsum, $size, $filename ) = @parts;
+                    push @parts, $hash;
                     if ($arch eq "source")
                     {
                         if ($component_regex)
                         {
+                            # Debian repository format https://wiki.debian.org/DebianRepository/Format#Debian_Repository_Format
                             if (
                                 (
-                                    $filename =~ m{^${component_regex}/source/}
+                                    $filename =~ m{^${component_regex}/source/Sources${compressed_extension_regex}}
                                 ) or (
-                                    $filename =~ m{^${component_regex}/Contents-source/}
+                                    $filename =~ m{^${component_regex}/Contents-source${compressed_extension_regex}}
                                 )
                             )
                             {
                                 push @parts_to_download, \@parts;
                             }
                         } else {
+                            # Flat repository format https://wiki.debian.org/DebianRepository/Format#Flat_Repository_Format
                             if ($filename =~ m{^Sources${compressed_extension_regex}}
                             ) {
                                 push @parts_to_download, \@parts;
                             }
                         }
                     } else {
-                        if (
-                            (
-                                $filename =~ m{^Contents-${arch_regex}${compressed_extension_regex}}
-                            ) or (
-                                $filename =~ m{^Packages${compressed_extension_regex}}
-                            ) or (
-                                $filename =~ m{^${component_regex}/Contents-${arch_regex}${compressed_extension_regex}}
-                            ) or (
-                                $filename =~ m{^${component_regex}/binary-${arch_regex}/Packages${compressed_extension_regex}}
-                            ) or (
-                                $filename =~ m{^${component_regex}/binary-${arch_regex}/Release$} # Needed for netboot.
-                            ) or (
-                                $filename =~ m{^${component_regex}/cnf/Commands-${arch_regex}${compressed_extension_regex}}
-                            ) or (
-                                $filename =~ m{^${component_regex}/dep11/Components-${arch_regex}.*${compressed_extension_regex}}
-                            ) or (
-                                $filename =~ m{^${component_regex}/dep11/icons-.*${compressed_extension_regex}}
-                            ) or (
-                                $filename =~ m{^${component_regex}/i18n/Translation-.*${compressed_extension_regex}}
-                            )
-                        )
+                        if ($component_regex)
                         {
-                            push @parts_to_download, \@parts;
+                            # Debian repository format https://wiki.debian.org/DebianRepository/Format#Debian_Repository_Format
+                            if (
+                                (
+                                    $filename =~ m{^${component_regex}/Contents-${arch_regex}${compressed_extension_regex}}
+                                ) or (
+                                    $filename =~ m{^Contents-${arch_regex}${compressed_extension_regex}}
+                                ) or (
+                                    $filename =~ m{^Packages${compressed_extension_regex}}
+                                ) or (
+                                    $filename =~ m{^${component_regex}/binary-${arch_regex}/Packages${compressed_extension_regex}}
+                                ) or (
+                                    $filename =~ m{^${component_regex}/binary-${arch_regex}/Release$}
+                                ) or (
+                                    $filename =~ m{^${component_regex}/cnf/Commands-${arch_regex}${compressed_extension_regex}}
+                                ) or (
+                                    $filename =~ m{^${component_regex}/dep11/Components-${arch_regex}.*${compressed_extension_regex}}
+                                ) or (
+                                    $filename =~ m{^${component_regex}/dep11/icons-.*${compressed_extension_regex}}
+                                ) or (
+                                    $filename =~ m{^${component_regex}/i18n/Translation-.*${compressed_extension_regex}}
+                                )
+                            )
+                            {
+                                push @parts_to_download, \@parts;
+                            }
+                        } else {
+                            # Flat repository format https://wiki.debian.org/DebianRepository/Format#Flat_Repository_Format
+                            if ($filename =~ m{^Packages${compressed_extension_regex}})
+                            {
+                                push @parts_to_download, \@parts;
+                            }
                         }
                     }
                 }
@@ -591,14 +633,15 @@ sub find_metadata_in_release
             }
             else
             {
-                $checksums = 0;
+                $hash = undef;
             }
         }
-        if ( not $checksums )
+        if ( not $hash )
         {
-            if ( $line eq "SHA256:" )
+            if ( $line =~ /^(${hash_type_regex}):$/ )
             {
-                $checksums = 1;
+                $hash = $1;
+                $avaiable_hashes{$hash} = 1;
             }
             elsif ( $line eq "Acquire-By-Hash: yes" )
             {
@@ -606,18 +649,39 @@ sub find_metadata_in_release
             }
         }
     }
+    close $stream;
+
+    my $strongest_hash;
+    if ($acquire_by_hash)
+    {
+        foreach (@hash_strength)
+        {
+            if ($avaiable_hashes{$_})
+            {
+                $strongest_hash = $_;
+                last;
+            }
+        }
+        unless ($strongest_hash)
+        {
+            warn("Cannot find a supported hash in $release_uri, will download from canonical locations.");
+            $acquire_by_hash = 0;
+        }
+    }
+
     foreach (@parts_to_download)
     {
-        my ( $sha256, $size, $filename ) = @{$_};
+        my ( $hashsum, $size, $filename, $hash ) = @{$_};
         if ($acquire_by_hash)
         {
-            add_url_to_download( $dist_uri . $filename, $size, $sha256 );
+            add_url_to_download( $dist_uri . $filename, $size, $strongest_hash, $hash, $hashsum );
         }
         else
         {
             add_url_to_download( $dist_uri . $filename, $size );
         }
     }
+    return 1;
 }
 
 print "Processing metadata files from releases [";
@@ -625,18 +689,73 @@ foreach (@config_binaries)
 {
     my ( $arch, $uri, $distribution, @components ) = @{$_};
     print "M";
-    find_metadata_in_release( $arch, $uri, $distribution, @components);
+    unless (find_metadata_in_release( $arch, $uri, $distribution, @components))
+    {
+        # Insecure repo with no release file - try to get the well known indices
+        foreach my $file_extension (".gz", ".bz2", ".xz", ".lzma", "")
+        {
+            if (@components)
+            {
+                # Debian repo
+                foreach my $component (@components)
+                {
+                    foreach my $path (
+                        "/dists/${distribution}/${component}/binary-${arch}/Packages",
+                        "/dists/${distribution}/${component}/binary-all/Packages",
+                        "/dists/${distribution}/${component}/Contents-${arch}",
+                        "/dists/${distribution}/${component}/Contents-all",
+                        "/dists/${distribution}/Contents-${arch}",
+                        "/dists/${distribution}/Contents-all",
+                    )
+                    {
+                        add_url_to_download( "${uri}/${path}${file_extension}" );
+                    }
+                }
+            } else {
+                # Flat repo
+                foreach my $path (
+                    "${distribution}/Packages",
+                    "${distribution}/Contents-${arch}",
+                    "${distribution}/Contents-all",
+                )
+                {
+                    add_url_to_download( "${uri}/${path}${file_extension}" );
+                }
+            }
+        }
+    }
 }
+
 foreach (@config_sources)
 {
     my ( $uri, $distribution, @components ) = @{$_};
     print "M";
-    find_metadata_in_release( "source", $uri, $distribution, @components);
+    unless (find_metadata_in_release( "source", $uri, $distribution, @components))
+    {
+        # Insecure repo with no release file - try to get the well known indices
+        foreach my $file_extension (".gz", ".bz2", ".xz", ".lzma", "")
+        {
+            if (@components)
+            {
+                # Debian repo
+                foreach my $path (
+                    "${distribution}/source/Sources",
+                    "${distribution}/Contents-source",
+                )
+                {
+                    add_url_to_download( "${uri}/${path}${file_extension}" );
+                }
+            } else {
+                # Flat repo
+                add_url_to_download( "${uri}/${distribution}/Sources${file_extension}" );
+            }
+        }
+    }
 }
 print "]\n\n";
 
-push( @index_urls, sort keys %urls_to_download );
-download_urls( "metadata", sort keys %urls_to_download );
+@index_urls = sort keys %urls_to_download;
+download_urls( "index", @index_urls );
 
 ######################################################################################
 ## Main download preparations
@@ -805,13 +924,9 @@ foreach (@config_binaries)
             process_index( $uri, "/dists/$distribution/$component/binary-all/Packages", 1 );
         }
     }
-    elsif ($distribution)
-    {
-        process_index( $uri, "/$distribution/Packages" );
-    }
     else
     {
-        process_index( $uri, "/Packages" );
+        process_index( $uri, "/$distribution/Packages" );
     }
 }
 
@@ -871,17 +986,27 @@ sub copy_file
     utime( $atime, $mtime, $to ) or die("apt-mirror: can't utime $to");
 }
 
-foreach (@index_urls)
+foreach (@release_urls, @index_urls)
 {
     die("apt-mirror: invalid url in index_urls") unless s[^(\w+)://][];
     copy_file( get_variable("skel_path") . "/" . sanitise_uri("$_"), get_variable("mirror_path") . "/" . sanitise_uri("$_") );
 
     my $sanitized_uri = sanitise_uri($_);
-    if ($sha256_filenames{$sanitized_uri})
+
+    # If we downloaded any files from a checksum location, now is the time to
+    # populate the canonical filename.
+    if ($hashsum_to_files{$sanitized_uri})
     {
-        foreach my $filename (@{$sha256_filenames{$sanitized_uri}})
+        foreach my $filename (@{$hashsum_to_files{$sanitized_uri}})
         {
             copy_file( get_variable("mirror_path") . "/" . $sanitized_uri, get_variable("mirror_path") . "/" . $filename );
+            if ($file_to_hashsums{$filename})
+            {
+                foreach my $hashsum_filename (@{$file_to_hashsums{$filename}})
+                {
+                    copy_file( get_variable("mirror_path") . "/" . $sanitized_uri, get_variable("mirror_path") . "/" . $hashsum_filename );
+                }
+            }
         }
     }
 }