Skip to content

Add netboot files #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
+45 −33
Closed

Add netboot files #21

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
918d37c
Add boost to deps to fix build with nix 2.x
lheckemann Jul 19, 2018
0cf0276
Make AWS upload optional
lheckemann Jul 21, 2018
96ada14
Add netboot files to channel
lheckemann Aug 1, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion default.nix
View file
Original file line number Diff line number Diff line change
@@ -15,7 +15,7 @@ stdenv.mkDerivation {
name = "nixos-channel-scripts";

buildInputs = with perlPackages;
[ pkgconfig nix sqlite makeWrapper perl FileSlurp LWP LWPProtocolHttps ListMoreUtils DBDSQLite NetAmazonS3 boehmgc nlohmann_json ];
[ pkgconfig nix sqlite makeWrapper perl FileSlurp LWP LWPProtocolHttps ListMoreUtils DBDSQLite NetAmazonS3 boehmgc nlohmann_json boost ];

buildCommand = ''
mkdir -p $out/bin
76 changes: 44 additions & 32 deletions mirror-nixos-branch.pl
View file
Original file line number Diff line number Diff line change
@@ -17,6 +17,7 @@

my $channelName = $ARGV[0];
my $releaseUrl = $ARGV[1];
my $useAWS = ($ENV{'AWS_DISABLE'} or "") eq "1";

die "Usage: $0 CHANNEL-NAME RELEASE-URL\n" unless defined $channelName && defined $releaseUrl;

@@ -31,19 +32,21 @@

$ENV{'GIT_DIR'} = "/home/hydra-mirror/nixpkgs-channels";

my $bucket;
if ($useAWS) {
# S3 setup.
my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die;
my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die;

# S3 setup.
my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die;
my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die;
my $s3 = Net::Amazon::S3->new(
{ aws_access_key_id => $aws_access_key_id,
aws_secret_access_key => $aws_secret_access_key,
retry => 1,
host => "s3-eu-west-1.amazonaws.com",
});

my $s3 = Net::Amazon::S3->new(
{ aws_access_key_id => $aws_access_key_id,
aws_secret_access_key => $aws_secret_access_key,
retry => 1,
host => "s3-eu-west-1.amazonaws.com",
});

my $bucket = $s3->bucket($bucketName) or die;
$bucket = $s3->bucket($bucketName) or die;
}


sub fetch {
@@ -80,7 +83,7 @@ sub fetch {
die "channel would go back in time from $curRelease to $releaseName, bailing out\n" if $d == 1;
}

if ($bucket->head_key("$releasePrefix")) {
if ($useAWS && $bucket->head_key("$releasePrefix")) {
print STDERR "release already exists\n";
} else {
my $tmpDir = "/data/releases/tmp/release-$channelName/$releaseName";
@@ -96,25 +99,27 @@ sub fetch {
}

sub downloadFile {
my ($jobName, $dstName) = @_;
my ($jobName, $dstBase) = @_;

my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));

my $srcFile = $buildInfo->{buildproducts}->{1}->{path} or die "job '$jobName' lacks a store path";
$dstName //= basename($srcFile);
my $dstFile = "$tmpDir/" . $dstName;

my $sha256_expected = $buildInfo->{buildproducts}->{1}->{sha256hash} or die;

if (! -e $dstFile) {
print STDERR "downloading $srcFile to $dstFile...\n";
write_file("$dstFile.sha256", "$sha256_expected $dstName");
system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$dstFile.tmp'") == 0
or die "unable to fetch $srcFile\n";
rename("$dstFile.tmp", $dstFile) or die;
}
my $buildproducts = $buildInfo->{buildproducts};
for my $product (values %$buildproducts) {
my %product = %$product;
my $srcFile = $product->{path} or die "job '$jobName' lacks a store path";
my $dstName = $dstBase // basename($srcFile);
my $dstFile = "$tmpDir/" . $dstName;

my $sha256_expected = $product->{sha256hash} or die "file '$srcFile' lacks an expected hash";

if (! -e $dstFile) {
print STDERR "downloading $srcFile to $dstFile...\n";
write_file("$dstFile.sha256", "$sha256_expected $dstName\n");
system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$dstFile.tmp'") == 0
or die "unable to fetch $srcFile\n";
rename("$dstFile.tmp", $dstFile) or die;
}

if (-e "$dstFile.sha256") {
my $sha256_actual = `nix hash-file --type sha256 '$dstFile'`;
chomp $sha256_actual;
if ($sha256_expected ne $sha256_actual) {
@@ -134,6 +139,11 @@ sub fetch {
#downloadFile("nixos.iso_graphical.i686-linux");
downloadFile("nixos.ova.x86_64-linux");
#downloadFile("nixos.ova.i686-linux");
# Netboot is currently not included in -small but maybe should be?
# This may fail because of nixos/hydra#580, hence wrapping in eval
# nixos/nixpkgs#44089 works around it, but is not backported to all channels.
eval { downloadFile("nixos.netboot.x86_64-linux"); };
warn "Could not download netboot files: $@" if $@;
}

} else {
@@ -175,7 +185,7 @@ sub fetch {
my $basename = basename $fn;
my $key = "$releasePrefix/" . $basename;

unless (defined $bucket->head_key($key)) {
if ($useAWS && ! (defined $bucket->head_key($key))) {
print STDERR "mirroring $fn to s3://$bucketName/$key...\n";
$bucket->add_key_filename(
$key, $fn,
@@ -196,11 +206,13 @@ sub fetch {

$html .= "</tbody></table></body></html>";

$bucket->add_key($releasePrefix, $html,
{ content_type => "text/html" })
or die $bucket->err . ": " . $bucket->errstr;
if ($useAWS) {
$bucket->add_key($releasePrefix, $html,
{ content_type => "text/html" })
or die $bucket->err . ": " . $bucket->errstr;

File::Path::remove_tree($tmpDir);
File::Path::remove_tree($tmpDir);
}
}

# Prevent concurrent writes to the channels directory.