Initial commit

Signed-off-by: Jo-Philipp Wich <jow@openwrt.org>

3 files changed, 649 insertions, 0 deletions

diff --git a/bin/make-index.pl b/bin/make-index.pl
new file mode 100755
index 0000000..a72b4d7
--- /dev/null
+++ b/bin/make-index.pl
@@ -0,0 +1,76 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+my $dir = $ARGV[0];
+
+die "Usage: $0 <package directory>\n" unless -d $dir;
+
+
+sub pkg_metadata
+{
+	my $file = shift || return;
+
+	return undef unless -f $file;
+
+	my $size = -s $file;
+	my $md5sum;
+	my $sha256sum;
+
+	if (open MD5, "md5sum $file |")
+	{
+		if (defined(my $line = readline MD5))
+		{
+			($md5sum) = $line =~ /^([0-9a-fA-F]{32})/;
+		}
+
+		close MD5;
+	}
+
+	if (open SHA256, "openssl dgst -sha256 $file |")
+	{
+		if (defined(my $line = readline SHA256))
+		{
+			($sha256sum) = $line =~ /([0-9a-fA-f]{64})$/;
+		}
+
+		close SHA256;
+	}
+
+	return undef unless $md5sum && $sha256sum;
+
+	my $meta = '';
+
+	if (open TAR, "tar -xzOf $file ./control.tar.gz | tar -xzOf - ./control |")
+	{
+		while (defined(my $line = readline TAR))
+		{
+			if ($line =~ /^Description:/)
+			{
+				$meta .= sprintf "Size: %d\n", $size;
+				$meta .= sprintf "MD5Sum: %s\n", $md5sum;
+				$meta .= sprintf "SHA256sum: %s\n", $sha256sum;
+			}
+
+			$meta .= $line;
+		}
+
+		$meta .= "\n";
+
+		close TAR;
+	}
+
+	return $meta;
+}
+
+if (opendir D, $dir)
+{
+	while (defined(my $e = readdir D))
+	{
+		next unless -f "$dir/$e" && $e =~ m{\.ipk$};
+		print pkg_metadata("$dir/$e");
+	}
+
+	closedir D;
+}
diff --git a/bin/patch-index.pl b/bin/patch-index.pl
new file mode 100755
index 0000000..317698d
--- /dev/null
+++ b/bin/patch-index.pl
@@ -0,0 +1,160 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+use Getopt::Long;
+
+my $index;
+my @add;
+my @remove;
+
+GetOptions(
+	"index=s"  => \$index,
+	"add=s"    => \@add,
+	"remove=s" => \@remove
+);
+
+
+unless (defined($index) && -f $index && (@add || @remove))
+{
+	die "Usage: $0 --index <pkg index> --add <pkg> --remove <pkg>\n";
+}
+
+sub pkg_basename
+{
+	my $file = shift || return;
+
+	return undef unless -f $file;
+
+	my ($name) = $file =~ m!^(?:.+/)?([^_/]+)[^/]+\.ipk$!;
+
+	return $name;
+}
+
+sub pkg_metadata
+{
+	my $file = shift || return;
+
+	return undef unless -f $file;
+
+	my $size = -s $file;
+	my $md5sum;
+	my $sha256sum;
+
+	if (open MD5, "md5sum $file |")
+	{
+		if (defined(my $line = readline MD5))
+		{
+			($md5sum) = $line =~ /^([0-9a-fA-F]{32})/;
+		}
+
+		close MD5;
+	}
+
+	if (open SHA256, "openssl dgst -sha256 $file |")
+	{
+		if (defined(my $line = readline SHA256))
+		{
+			($sha256sum) = $line =~ /([0-9a-fA-f]{64})$/;
+		}
+
+		close SHA256;
+	}
+
+	return undef unless $md5sum && $sha256sum;
+
+	my $meta = '';
+	my ($name) = $file =~ m!([^/]+)$!;
+
+	if (open TAR, "tar -xzOf $file ./control.tar.gz | tar -xzOf - ./control |")
+	{
+		while (defined(my $line = readline TAR))
+		{
+			if ($line =~ /^Description:/)
+			{
+				$meta .= sprintf "Filename: %s\n", $name;
+				$meta .= sprintf "Size: %d\n", $size;
+				$meta .= sprintf "MD5Sum: %s\n", $md5sum;
+				$meta .= sprintf "SHA256sum: %s\n", $sha256sum;
+			}
+
+			$meta .= $line;
+		}
+
+		$meta .= "\n";
+
+		close TAR;
+	}
+
+	# Fix up source
+	$meta =~ s!^Source: .+/(feeds/.+)$!Source: $1!m;
+
+	return $meta;
+}
+
+
+my %addpkg;
+my %rempkg;
+
+foreach my $pkg (@add)
+{
+	my $name = pkg_basename($pkg);
+	my $meta = pkg_metadata($pkg);
+
+	if (!defined $name || !defined $meta)
+	{
+		warn "$pkg is not a valid .ipk file\n";
+		next;
+	}
+
+	$addpkg{$name} = $meta;
+	$rempkg{$name} = 1;
+}
+
+my ($cur_pkg, $cur_meta);
+my $cat = ($index =~ /\.gz$/) ? "zcat" : "cat";
+
+@rempkg{@remove} = (1) x @remove;
+
+if (open IDX, "$cat $index |")
+{
+	while (1)
+	{
+		my $line = readline IDX;
+
+		if (defined($line) && $line =~ /^Package: (.+)$/)
+		{
+			$cur_pkg = $1;
+			$cur_meta = $line;
+		}
+		elsif (defined($line) && $line !~ /^$/ && defined($cur_meta))
+		{
+			$cur_meta .= $line;
+		}
+		else
+		{
+			foreach my $add (sort keys %addpkg)
+			{
+				if (!defined($cur_pkg) || $add le $cur_pkg)
+				{
+					print $addpkg{$add};
+					delete $addpkg{$add};
+
+					#$cur_pkg = $add;					
+				}
+			}
+			
+			if (defined($cur_pkg) && !$rempkg{$cur_pkg})
+			{
+				print $cur_meta, "\n";
+			}
+
+			undef $cur_pkg;
+			undef $cur_meta;
+		}
+
+		last unless defined $line;
+	}
+
+	close IDX;
+}
diff --git a/pkgupdate.sh b/pkgupdate.sh
new file mode 100755
index 0000000..532be3e
--- /dev/null
+++ b/pkgupdate.sh
@@ -0,0 +1,413 @@
+#!/bin/bash
+
+
+#U="https://downloads.openwrt.org/barrier_breaker/14.07"
+U="file:///BB/sync/barrier_breaker/14.07"
+R="user@remote:/foo/barrier_breaker/14.07"
+T="$(pwd)"
+
+tmp="/home/jow/.relman"
+
+mkdir -p "$tmp"
+
+terminate() {
+	local jobs="$(jobs -p)"
+
+	echo "Terminating..."
+
+	pkill -P $$
+	exit 1
+}
+
+fetch_remote() {
+	if [ -f "${1#file:}" ]; then
+		cat "${1#file:}"
+	else
+		wget -qO- "$1"
+	fi
+}
+
+fetch_remote_dirlist() {
+	local entry
+
+	if [ -d "${1#file:}" ]; then
+		/bin/ls -1 "${1#file:}" | while read entry; do
+			if [ -d "${1#file:}/$entry" ]; then
+				echo "$entry"
+			fi
+		done
+	else
+		wget -qO- "$1" | sed -ne 's,^<a href="\(.\+\)/".\+$,\1,p'
+	fi
+}
+
+fetch_remote_filelist() {
+	local entry
+
+	if [ -d "${1#file:}" ]; then
+		/bin/ls -1 "${1#file:}" | while read entry; do
+			if [ -f "${1#file:}/$entry" ]; then
+				echo "$entry"
+			fi
+		done
+	else
+		wget -qO- "$1" | sed -ne 's,^<a href="\(.\+[^/]\)".\+$,\1,p'
+	fi
+}
+
+fetch_remote_targets() {
+	local target subtarget
+
+	if [ -n "$use_targets" ]; then
+		for target in $use_targets; do
+			echo "$target"
+		done
+		return 0
+	fi
+
+	if [ ! -s "$tmp/targets.lst" ]; then
+		fetch_remote_dirlist "$U" | while read target; do
+			[ "$target" = "logs" ] && continue
+			fetch_remote_dirlist "$U/$target" | while read subtarget; do
+				echo "$target/$subtarget" >> "$tmp/targets.lst"
+			done
+		done
+	fi
+
+	cat "$tmp/targets.lst"
+	return 0
+}
+
+fetch_remote_feeds() {
+	local target
+
+	if [ ! -s "$tmp/feeds.lst" ]; then
+		fetch_remote_targets | while read target; do
+			fetch_remote_dirlist "$U/$target/packages" > "$tmp/feeds.lst"
+			break
+		done
+	fi
+
+	cat "$tmp/feeds.lst"
+	return 0
+}
+
+fetch_remote_index() {
+	local target feed
+
+	echo "Fetching remote package indizes..."
+
+	fetch_remote_targets | while read target; do
+		fetch_remote_feeds | while read feed; do
+			if [ ! -s "$tmp/repo-remote/$target/packages/$feed/Packages.gz" ]; then
+				echo " * $target $feed"
+				mkdir -p "$tmp/repo-remote/$target/packages/$feed"
+				wget -qO "$tmp/repo-remote/$target/packages/$feed/Packages.gz" "$U/$target/packages/$feed/Packages.gz"
+			fi
+		done
+	done
+}
+
+fetch_remote_sdk() {
+	local target sdk
+
+	echo "Fetching remote SDKs..."
+
+	fetch_remote_targets | while read target; do
+		if [ ! -s "$tmp/repo-remote/$target/sdk.tar.bz2" ]; then
+			fetch_remote_filelist "$U/$target" | while read sdk; do
+				case "$sdk" in OpenWrt-SDK-*.tar.bz2)
+					echo " * $target $sdk"
+					mkdir -p "$tmp/repo-remote/$target"
+					fetch_remote "$U/$target/$sdk" > "$tmp/repo-remote/$target/sdk.tar.bz2"
+				;; esac
+			done
+		fi
+	done
+}
+
+sdk_target() {
+	sed -ne 's,^CONFIG_TARGET_BOARD="\(.*\)",\1,p' "$T/.config"
+}
+
+sdk_subtarget() {
+	local mk target subtarget
+	for mk in "$T/target/linux"/*/*/target.mk; do
+		mk="${mk#*/target/linux/}"
+		target="${mk%/*/target.mk}"
+		subtarget="${mk#$target/}"
+		subtarget="${subtarget%/target.mk}"
+
+		if grep -qE "^CONFIG_TARGET_${target}_${subtarget}=y\$" "$T/.config"; then
+			echo "$subtarget"
+			return 0
+		fi
+	done
+	return 1
+}
+
+sdk_prepare() {
+	local target="$1"
+
+	if [ ! -d "$tmp/sdk/$target/.git" ]; then
+		echo " * [$slot:$target] Initializing SDK"
+
+		rm -rf "$tmp/sdk/$target"
+		mkdir -p "$tmp/sdk/$target"
+		tar --strip-components=1 -C "$tmp/sdk/$target" -xjf "$tmp/repo-remote/$target/sdk.tar.bz2"
+
+		mkdir -p "$tmp/dl"
+		rm -rf "$tmp/sdk/$target/dl"
+		ln -sf "$tmp/dl" "$tmp/sdk/$target/dl"
+
+		mkdir -p "$tmp/feeds"
+		rm -rf "$tmp/sdk/$target/feeds"
+		ln -sf "$tmp/feeds" "$tmp/sdk/$target/feeds"
+
+		(
+			cd "$tmp/sdk/$target"
+			git init .
+			find . -maxdepth 1 | xargs git add
+			git commit -m "Snapshot"
+		) >/dev/null
+	else
+		echo " * [$slot:$target] Resetting SDK"
+
+		(
+			cd "$tmp/sdk/$target"
+			git reset --hard HEAD
+			git clean -f -d
+		) >/dev/null
+	fi
+}
+
+sdk_feed_install() {
+	local pkg feed target="$1"; shift
+
+	echo " * [$slot:$target] Installing packages"
+
+	(
+		flock -x 9
+
+		cd "$tmp/sdk/$target"
+
+		./scripts/feeds update
+
+		for pkg in "$@"; do
+			case "$pkg" in
+				*:*) feed="${pkg#*:}"; pkg="${pkg%%:*}" ;;
+				*) feed="" ;;
+			esac
+
+			./scripts/feeds install -d m${feed:+ -p "$feed"} "$pkg"
+		done
+	) 9>"$tmp/feeds.lock" 2>/dev/null >/dev/null
+}
+
+sdk_pkg_compile() {
+	local pkg target="$1"; shift
+
+	echo " * [$slot:$target] Compiling packages"
+
+	for pkg in "$@"; do
+		(cd "$tmp/sdk/$target"; make "package/${pkg%%:*}/compile") >/dev/null
+
+		mkdir -p "$tmp/repo-local/$target/packages"
+		cp -a "$tmp/sdk/$target/bin"/*/packages/* "$tmp/repo-local/$target/packages/"
+	done
+}
+
+find_remote_pkg_name() {
+	local feed name target="$1" pkg="$2"
+
+	while read feed; do
+		name="$(zcat "$tmp/repo-remote/$target/packages/$feed/Packages.gz" | \
+			sed -ne "s/Filename: \\(${pkg%%:*}_.\\+\\.ipk\\)\$/\1/p")"
+
+		if [ -n "$name" ]; then
+			echo "$target/packages/$feed/$name"
+			return 0
+		fi
+	done < "$tmp/feeds.lst"
+
+	return 1
+}
+
+find_remote_pkg_feed() {
+	local feed target="$1" pkg="$2"
+
+	while read feed; do
+		if zcat "$tmp/repo-remote/$target/packages/$feed/Packages.gz" | grep -qE "^Package: ${pkg%%:*}\$"; then
+			echo "$feed"
+			return 0
+		fi
+	done < "$tmp/feeds.lst"
+
+	return 1
+}
+
+find_local_pkg_feed() {
+	local feed file target="$1" pkg="$2"
+
+	while read feed; do
+		for file in "$tmp/repo-local/$target/packages/$feed/${pkg%%:*}"_[^_]*_[^_]*.ipk; do
+			if [ -s "$file" ]; then
+				echo "$feed"
+				return 0
+			fi
+		done
+	done < "$tmp/feeds.lst"
+
+	return 1
+}
+
+index_patch_cmd() {
+	local target="$1" feed="$2"; shift; shift
+	local idir="$tmp/repo-remote/$target/packages/$feed"
+	local odir="$tmp/repo-local/$target/packages/$feed"
+
+	if [ ! -s "$odir/Packages" ]; then
+		mkdir -p "$odir"
+		zcat "$idir/Packages.gz" > "$odir/Packages"
+	fi
+
+	./bin/patch-index.pl --index "$odir/Packages" "$@" > "$odir/Packages.$$"
+
+	mv "$odir/Packages.$$" "$odir/Packages"
+}
+
+index_patch() {
+	local target="$1" feed pkg dir; shift
+
+	echo " * [$slot:$target] Patching repository index"
+
+	for pkg in "$@"; do
+		feed="$(find_remote_pkg_feed "$target" "$pkg")"
+		[ -n "$feed" ] && index_patch_cmd "$target" "$feed" \
+			--remove "${pkg%%:*}"
+
+		feed="$(find_local_pkg_feed "$target" "$pkg")"
+		[ -n "$feed" ] && index_patch_cmd "$target" "$feed" \
+			--add "$tmp/repo-local/$target/packages/$feed/${pkg%%:*}"_*.ipk
+	done
+
+	while read feed; do
+		dir="$tmp/repo-local/$target/packages/$feed"
+		if [ -s "$dir/Packages" ]; then
+			gzip -c -9 "$dir/Packages" > "$dir/Packages.gz"
+		fi
+	done < "$tmp/feeds.lst"
+}
+
+file_rsync() {
+	local target="$1" pkg path; shift
+
+	echo " * [$slot:$target] Syncing files"
+
+	mkdir -p "$tmp/empty"
+
+	for pkg in "$@"; do
+		path="$(find_remote_pkg_name "$target" "$pkg")"
+
+		[ -n "$path" ] && echo rsync --dry-run -rv \
+			--delete="${path##*/}" --exclude="*" "$tmp/empty/" "$R/${path%/*}/"
+
+		echo rsync --dry-run -rv "$tmp/repo-local/$target/packages/" "$R/$target/packages/"
+	done
+
+}
+
+run_jobs() {
+	local targets=$(fetch_remote_targets)
+	local target slot count
+
+	#echo "* Compiling packages"
+
+	for slot in $(seq 0 $((num_jobs-1))); do (
+		count=1; for target in $targets; do
+			if [ $((count++ % $num_jobs)) -eq $slot ]; then
+				if [ $do_compile -gt 0 ]; then
+					sdk_prepare "$target"
+					sdk_feed_install "$target" "$@"
+					sdk_pkg_compile "$target" "$@"
+				fi
+
+				if [ $do_index -gt 0 ]; then
+					index_patch "$target" "$@"
+				fi
+
+				if [ $do_rsync -gt 0 ]; then
+					file_rsync "$target" "$@"
+				fi
+			fi
+		done
+	) & done
+}
+
+trap terminate INT TERM
+
+do_compile=0
+do_index=0
+do_rsync=0
+
+num_jobs=$(grep processor /proc/cpuinfo | wc -l)
+
+use_targets=""
+use_packages=""
+
+usage() {
+	cat <<-EOT
+$0 {-a|-c|-i|-r} [-j jobs] -p package [-p package ...]
+
+  -a
+     Perform all build steps (compile, rebuild, rsync)
+
+  -c
+     Compile packages for all architectures.
+
+  -i
+     Rebuild indexes for all architectures.
+
+  -r
+     Rsync files for all architecures.
+
+  -j jobs
+     Use the given number of jobs when compiling packages.
+     Default is $num_jobs jobs.
+
+  -p package[:feed]
+     Add given package to the queue. If a feed suffix is
+     given then prefer this feed name when installing the
+     package into the SDK.
+
+
+  -t target
+     Restrict operation to given targets.
+
+	EOT
+	exit 1
+}
+
+while getopts ":acirp:t:" opt; do
+	case "$opt" in
+		a)
+			do_compile=1
+			do_index=1
+			do_rsync=1
+		;;
+		c) do_compile=1 ;;
+		i) do_index=1 ;;
+		r) do_rsync=1 ;;
+		j) num_jobs="$OPTARG" ;;
+		p) use_packages="${use_packages:+$use_packages }$OPTARG" ;;
+		t) use_targets="${use_targets:+$use_targets }$OPTARG" ;;
+	esac
+done
+
+[ -z "$use_packages" ] && usage
+
+echo "* Preparing metadata"
+fetch_remote_targets >/dev/null
+fetch_remote_feeds >/dev/null
+
+run_jobs "$use_packages"