diff options
author | Sean Whitton <spwhitton@spwhitton.name> | 2018-07-19 01:09:16 +0800 |
---|---|---|
committer | Sean Whitton <spwhitton@spwhitton.name> | 2018-07-19 01:09:16 +0800 |
commit | d0492d8f40b05ea5f4d378cd34221a9cd66cbfcd (patch) | |
tree | d79f415f344a81aca8d18e916b7016a84aec5793 | |
parent | eb07108c1051f227c83de01781f498a330faae43 (diff) | |
parent | 15fbfc94b8c6a08cffdf0f9a7ed5870252ad4416 (diff) |
Merge tag 'debian/5.10' into stretch-bpo
dgit release 5.10 for unstable (sid) [dgit]
[dgit distro=debian]
# gpg: Signature made Sun 15 Jul 2018 01:12:02 AM CST
# gpg: using RSA key 559AE46C2D6B6D3265E7CBA1E3E3392348B50D39
# gpg: Can't check signature: No public key
-rw-r--r-- | .gitignore | 1 | ||||
-rw-r--r-- | Debian/Dgit.pm | 83 | ||||
-rw-r--r-- | Makefile | 1 | ||||
-rw-r--r-- | NOTES.dgit-downstream-dsc.7.pod | 69 | ||||
-rw-r--r-- | debian/changelog | 35 | ||||
-rwxr-xr-x | dgit | 78 | ||||
-rw-r--r-- | dgit-downstream-dsc.7.pod | 354 | ||||
-rw-r--r-- | dgit.1 | 1 | ||||
-rwxr-xr-x | git-debrebase | 107 | ||||
-rw-r--r-- | git-debrebase.1.pod | 9 | ||||
-rw-r--r-- | git-debrebase.5.pod | 7 | ||||
-rwxr-xr-x | infra/dgit-mirror-rsync | 7 | ||||
-rw-r--r-- | tests/lib | 21 | ||||
-rw-r--r-- | tests/lib-gdr | 2 | ||||
-rwxr-xr-x | tests/playtree-save-refs | 9 | ||||
-rwxr-xr-x | tests/setup/examplegit | 2 | ||||
-rwxr-xr-x | tests/setup/gdr-convert-gbp | 8 | ||||
-rwxr-xr-x | tests/setup/gnupg | 2 | ||||
-rwxr-xr-x | tests/tests/gdr-diverge-nmu-dgit | 2 | ||||
-rwxr-xr-x | tests/tests/gdr-edits | 2 | ||||
-rwxr-xr-x | tests/tests/gdr-import-dgit | 2 |
21 files changed, 699 insertions, 103 deletions
@@ -14,6 +14,7 @@ dgit-maint-merge.7 dgit-maint-gbp.7 dgit-maint-debrebase.7 dgit-sponsorship.7 +dgit-downstream-dsc.7 git-debrebase.1 git-debrebase.5 substituted diff --git a/Debian/Dgit.pm b/Debian/Dgit.pm index 5106f85..0e2464d 100644 --- a/Debian/Dgit.pm +++ b/Debian/Dgit.pm @@ -31,6 +31,7 @@ use Data::Dumper; use IPC::Open2; use File::Path; use File::Basename; +use Dpkg::Control::Hash; BEGIN { use Exporter (); @@ -41,6 +42,7 @@ BEGIN { @EXPORT = qw(setup_sigwarn forkcheck_setup forkcheck_mainprocess dep14_version_mangle debiantags debiantag_old debiantag_new + debiantag_maintview server_branch server_ref stat_exists link_ltarget hashfile @@ -66,6 +68,8 @@ BEGIN { $negate_harmful_gitattrs changedir git_slurp_config_src gdr_ffq_prev_branchinfo + parsecontrolfh parsecontrol parsechangelog + getfield parsechangelog_loop playtree_setup); # implicitly uses $main::us %EXPORT_TAGS = ( policyflags => [qw(NOFFCHECK FRESHREPO NOCOMMITCHECK)], @@ -206,6 +210,11 @@ sub debiantag_new ($$) { return "archive/$distro/".dep14_version_mangle $v; } +sub debiantag_maintview ($$) { + my ($v,$distro) = @_; + return "$distro/".dep14_version_mangle $v; +} + sub debiantags ($$) { my ($version,$distro) = @_; map { $_->($version, $distro) } (\&debiantag_new, \&debiantag_old); @@ -554,6 +563,80 @@ sub gdr_ffq_prev_branchinfo ($) { return ('branch', undef, $symref, $ffq_prev, $gdrlast); } +sub parsecontrolfh ($$;$) { + my ($fh, $desc, $allowsigned) = @_; + our $dpkgcontrolhash_noissigned; + my $c; + for (;;) { + my %opts = ('name' => $desc); + $opts{allow_pgp}= $allowsigned || !$dpkgcontrolhash_noissigned; + $c = Dpkg::Control::Hash->new(%opts); + $c->parse($fh,$desc) or die "parsing of $desc failed"; + last if $allowsigned; + last if $dpkgcontrolhash_noissigned; + my $issigned= $c->get_option('is_pgp_signed'); + if (!defined $issigned) { + $dpkgcontrolhash_noissigned= 1; + seek $fh, 0,0 or die "seek $desc: $!"; + } elsif ($issigned) { + fail "control file $desc is (already) PGP-signed. ". + " Note that dgit push needs to modify the .dsc and then". + " do the signature itself"; + } else { + last; + } + } + return $c; +} + +sub parsecontrol { + my ($file, $desc, $allowsigned) = @_; + my $fh = new IO::Handle; + open $fh, '<', $file or die "$file: $!"; + my $c = parsecontrolfh($fh,$desc,$allowsigned); + $fh->error and die $!; + close $fh; + return $c; +} + +sub parsechangelog { + my $c = Dpkg::Control::Hash->new(name => 'parsed changelog'); + my $p = new IO::Handle; + my @cmd = (qw(dpkg-parsechangelog), @_); + open $p, '-|', @cmd or die $!; + $c->parse($p); + $?=0; $!=0; close $p or failedcmd @cmd; + return $c; +} + +sub getfield ($$) { + my ($dctrl,$field) = @_; + my $v = $dctrl->{$field}; + return $v if defined $v; + fail "missing field $field in ".$dctrl->get_option('name'); +} + +sub parsechangelog_loop ($$$) { + my ($clogcmd, $descbase, $fn) = @_; + # @$clogcmd is qw(dpkg-parsechangelog ...some...options...) + # calls $fn->($thisstanza, $desc); + debugcmd "|",@$clogcmd; + open CLOGS, "-|", @$clogcmd or die $!; + for (;;) { + my $stanzatext = do { local $/=""; <CLOGS>; }; + printdebug "clogp stanza ".Dumper($stanzatext) if $debuglevel>1; + last if !defined $stanzatext; + + my $desc = "$descbase, entry no.$."; + open my $stanzafh, "<", \$stanzatext or die; + my $thisstanza = parsecontrolfh $stanzafh, $desc, 1; + + $fn->($thisstanza, $desc); + } + die $! if CLOGS->error; + close CLOGS or $?==SIGPIPE or failedcmd @$clogcmd; +} + # ========== playground handling ========== # terminology: @@ -41,6 +41,7 @@ MAN7PAGES=dgit.7 \ dgit-maint-native.7 \ dgit-maint-merge.7 dgit-maint-gbp.7 \ dgit-maint-debrebase.7 \ + dgit-downstream-dsc.7 \ dgit-sponsorship.7 TXTDOCS=README.dsc-import diff --git a/NOTES.dgit-downstream-dsc.7.pod b/NOTES.dgit-downstream-dsc.7.pod new file mode 100644 index 0000000..9be7cc3 --- /dev/null +++ b/NOTES.dgit-downstream-dsc.7.pod @@ -0,0 +1,69 @@ +NOTE This text was once going to be part of dgit-downstream-dsc(7) or + dgit-downstream-dsc(5). It probably wants to be reworked, and + maybe put there, to fix + #810829 want instructions for reprepro-style small repo + +This guide is to help you if: + + * you are a distro which is a downstream of Debian (directly + or indirectly) + + * you want to publish source packages as well as git branches + +You will also need: + + * A git server. [...] + + There are various options for the git server, depending on how much + you trust your uploaders. There are four levels of trust and + sophistication: + + shell account + + For use when uploaders have shell accounts on the server and you + trust them completely. You then do not need to install any special + software on the server. + + dgit-repos-server + + Your uploaders do not (necessarily) have shell accounts. + You will need to collect their ssh keys and also their PGP + signing keys. You can restrict uploads on a per-package + per-key basis by using the Debian `dm.txt' format. + + dgit-repos-server + policy hook + + You want to impose additional policy. For example, Debian's + copyright review process means that uploads of new packages are + initially not public: dgit-repos-policy-debian is an example. + + custom implementation + + From the dgit client's point of view, the dgit git server is a git + server accessed by ssh (when pushing) or https (when fetching). + You may use anything that has the right properties for your needs. + dgit primarily authenticates pushes by signing tags, so your + software will probably need to check and verify that tag + appropriately before accepting a push. dgit-repos-server knows how + to do this properly. + +Set up your git server, as follows: + + shell account + + Make a suitable (sub)directory. You should create a _template.git + bare repo, with appropriate permissions. When new packages are + uploaded, this _template.git will be copied. You will probably + want to set core.sharedRepository in the template, and/or arrange + for personal groups and 002 umask. + + dgit-repos-server + + Additionally, install dgit-infrastructure. Create a service + account `dgit' on the server. For each authorised uploader, put + their ssh key in dgit's authorized_keys file, with a + restricted_command specifying the dgit-repos-server invocation. + Put the keyring where dgit-repos-server can find it. + Consult the comment at the top of dgit-repos-server for the + restricted command rune. + diff --git a/debian/changelog b/debian/changelog index 2239a69..a7b12d8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,38 @@ +dgit (5.10) unstable; urgency=medium + + * Merge the experimental branch. + * test suite: Drop a couple of useless test log output lines. + * infrastructure: Run git gc --auto before mirroring. Closes:#841414. + + -- Ian Jackson <ijackson@chiark.greenend.org.uk> Sat, 14 Jul 2018 18:07:02 +0100 + +dgit (5.9+exp4) experimental; urgency=medium + + * test suite: Use dch -r -D sid '' not dch -r sid. Closes:#903441. + * test suite: Save a tarball of much of the working area of each test + in $AUTOPKGTEST_ARTIFACTS. + * Separate changelog entries for the following test attempts + in experimental have been elided: + dgit (5.9+exp3) experimental; urgency=medium + dgit (5.9+exp2) experimental; urgency=medium + dgit (5.9+exp1) experimental; urgency=medium + + -- Ian Jackson <ijackson@chiark.greenend.org.uk> Thu, 12 Jul 2018 13:45:07 +0100 + +dgit (5.9) unstable; urgency=medium + + * git-debrebase convert-from-gbp: Look for dgit-generated tags so we can + usually make the new branch ff of the dgit view. Closes:#903132. + * git-debrebase convert-from-gbp: Check that the result will not + count as having diverged. This will usually turn failures to make + the ff pseudomerge into -fdiverged. Related to #903132. + * git-debrebase, Dgit.pm, git: some internal reorganisation to + support git-debrebase changes. + * dgit-downstream-dsc(7): New manpage. Closes:#842643,#851194. + * git-debrebase(5): Document best gitk options. Closes:#901927. + + -- Ian Jackson <ijackson@chiark.greenend.org.uk> Thu, 12 Jul 2018 13:37:12 +0100 + dgit (5.8~bpo9+1) stretch-backports; urgency=medium * Rebuild for stretch-backports. @@ -189,11 +189,6 @@ sub debiantag ($$) { return $tagformatfn->($v, $distro); } -sub debiantag_maintview ($$) { - my ($v,$distro) = @_; - return "$distro/".dep14_version_mangle $v; -} - sub madformat ($) { $_[0] eq '3.0 (quilt)' } sub lbranch () { return "$branchprefix/$csuite"; } @@ -962,59 +957,6 @@ sub access_giturl (;$) { return "$url/$package$suffix"; } -sub parsecontrolfh ($$;$) { - my ($fh, $desc, $allowsigned) = @_; - our $dpkgcontrolhash_noissigned; - my $c; - for (;;) { - my %opts = ('name' => $desc); - $opts{allow_pgp}= $allowsigned || !$dpkgcontrolhash_noissigned; - $c = Dpkg::Control::Hash->new(%opts); - $c->parse($fh,$desc) or die "parsing of $desc failed"; - last if $allowsigned; - last if $dpkgcontrolhash_noissigned; - my $issigned= $c->get_option('is_pgp_signed'); - if (!defined $issigned) { - $dpkgcontrolhash_noissigned= 1; - seek $fh, 0,0 or die "seek $desc: $!"; - } elsif ($issigned) { - fail "control file $desc is (already) PGP-signed. ". - " Note that dgit push needs to modify the .dsc and then". - " do the signature itself"; - } else { - last; - } - } - return $c; -} - -sub parsecontrol { - my ($file, $desc, $allowsigned) = @_; - my $fh = new IO::Handle; - open $fh, '<', $file or die "$file: $!"; - my $c = parsecontrolfh($fh,$desc,$allowsigned); - $fh->error and die $!; - close $fh; - return $c; -} - -sub getfield ($$) { - my ($dctrl,$field) = @_; - my $v = $dctrl->{$field}; - return $v if defined $v; - fail "missing field $field in ".$dctrl->get_option('name'); -} - -sub parsechangelog { - my $c = Dpkg::Control::Hash->new(name => 'parsed changelog'); - my $p = new IO::Handle; - my @cmd = (qw(dpkg-parsechangelog), @_); - open $p, '-|', @cmd or die $!; - $c->parse($p); - $?=0; $!=0; close $p or failedcmd @cmd; - return $c; -} - sub commit_getclogp ($) { # Returns the parsed changelog hashref for a particular commit my ($objid) = @_; @@ -2342,22 +2284,14 @@ sub generate_commits_from_dsc () { } my @clogcmd = qw(dpkg-parsechangelog --format rfc822 --all); - debugcmd "|",@clogcmd; - open CLOGS, "-|", @clogcmd or die $!; - my $clogp; my $r1clogp; printdebug "import clog search...\n"; + parsechangelog_loop \@clogcmd, "package changelog", sub { + my ($thisstanza, $desc) = @_; + no warnings qw(exiting); - for (;;) { - my $stanzatext = do { local $/=""; <CLOGS>; }; - printdebug "import clogp ".Dumper($stanzatext) if $debuglevel>1; - last if !defined $stanzatext; - - my $desc = "package changelog, entry no.$."; - open my $stanzafh, "<", \$stanzatext or die; - my $thisstanza = parsecontrolfh $stanzafh, $desc, 1; $clogp //= $thisstanza; printdebug "import clog $thisstanza->{version} $desc...\n"; @@ -2383,7 +2317,7 @@ sub generate_commits_from_dsc () { # version). Then it remains to choose between the physically # last entry in the file, and the one with the lowest version # number. If these are not the same, we guess that the - # versions were created in a non-monotic order rather than + # versions were created in a non-monotonic order rather than # that the changelog entries have been misordered. printdebug "import clog $thisstanza->{version} vs $upstreamv...\n"; @@ -2392,9 +2326,7 @@ sub generate_commits_from_dsc () { $r1clogp = $thisstanza; printdebug "import clog $r1clogp->{version} becomes r1\n"; - } - die $! if CLOGS->error; - close CLOGS or $?==SIGPIPE or failedcmd @clogcmd; + }; $clogp or fail "package changelog has no entries!"; diff --git a/dgit-downstream-dsc.7.pod b/dgit-downstream-dsc.7.pod new file mode 100644 index 0000000..fcbce05 --- /dev/null +++ b/dgit-downstream-dsc.7.pod @@ -0,0 +1,354 @@ +=head1 NAME + +dgit-downstream-dsc - setting up dgit push for a new distro + +=head1 INTRODUCTION + +This document is aimed at downstreams of Debian. +It explains how you can +publish your packages' source code +both as traditional Debian source packages, +and as git branches, +using B<dgit push>. +Your users will be able to get the source with B<dgit clone>, +or with traditional tools such as B<apt-get source>. + +Note that often it is unnecessary to +publish traditional source packages. +Debian-format source packages can be complex, +idiosyncratic and difficult to work with. +You should avoid them if you can. +If you do not need to publish source packages, +you can work as a Debian downstream purely using git branches, +(using dgit to get the source from Debian in git form). +You can build binaries directly from git, +and push package source code as a git branch to an ordinary git server. +See L<dgit-user(7)>. + +Not every option is covered here. +L<dgit(1)> has a mostly-complete list +of config options, although not always with useful descriptions. + +=head1 NAMES + +You need to choose some names. + +=over + +=item I<distro> name + +dgit understands each thing it interacts with as a B<distro>. +So in dgit terms, you are setting up a distro. + +You need a name for yourself (ie for your distro). +The name will appear in the git tags made by your tools, +and in configuration settings. +It must be globally unique across +all people and institutions who use dgit. + +You could choose your organisation's domain name, +or a part of it if you think that is going to be very unique. + +The distro name may contain ascii alphanumerics and B<. + ->, +although B<-> may be confusing and is probably best avoided. +Try to avoid uppercase letters (and underscore): +you will be typing this name a lot. + +For example, +if you were the Free Software Foundation Europe (fsfe.org) +you might call your distro fsfe or fsfe.org. +In the rest of this document we will write +I<distro> for your distro name. + +=item I<suite> names + +In dgit and Debian archive terminology, a B<suite> is a line of +development, and/or a Debian release. +For example, at the time of writing, Debian has suites like +B<sid> aka B<unstable>, B<buster> aka B<testing>, +and B<stretch> aka B<stable>. +There are also ancillary suites like B<stretch-security>. + +If your releases align with Debian's releases, +then your suites should contain the Debian suite names. +B<Do not> use just the Debian names. +That will cause confusion. +Instead, prepend your organisation's name and a hyphen. +For example, FSFE might end up with suites like fsfe-stretch. + +Suite names end up in git ref and branch names, +and on dgit command lines. +Suite names can contain alphanumerics and C<->. +Other characters may work but are not recommended. + +=back + +=head1 SERVICES + +You will need to run two parallel services: + +=over + +=item git server + +This will hold the git branches accessed by dgit. + +Everyone who will use dgit push needs to be able to update +B<refs/dgit/>I<suite> +(note, not B<refs/heads/dgit/>I<suite>) on that server, +and to make tags +I<distro>B</>I<version> and B<archive/>I<distro>B</>I<version>. +Normally this would be done over ssh. + +The server may host other branches and tags too. +So this might be your ordinary git server, +or an instance of a git hosting system. + +Everyone who obtains one of your source packages, +or who will run B<dgit clone> and B<dgit fetch>, +needs to have at least read access to the git server. +Ideally everything would be published +via the git smart https protocol. + +The git server name, and public git url structure, +should be chosen so they will not need to change in the future. +Best is to give the git server a DNS name of its own. + +Debian's dgit git server has special access control rules, +implemented in B<dgit-repos-server> and B<dgit-repos-policy-debian> +in the package B<dgit-infrastructure>. +but in most installations this is not needed. +If there is no or little distinction between +(i) developers who are entitled to upload (push) and +(ii) repository administrators, +then a it is sufficient to provide a +git server with a unix account for each user who will pushing, +perhaps using ssh restricted commands. + +=item Debian-format archive (repository) + +This holds the source packages. +You will probably use the same archive to host your binaries, +and point your B<apt> at it. + +dgit uses the term B<archive> for this. + +There are a variety of tools for +creating and managing a Debian-format archive. +In this document we will assume you are using B<reprepro>. + +Setting up reprepro is not covered in this tutorial. +Instead, we assume you already have reprepro working. + +You should also write appropriate dput configuration, +since dgit uses dput to upload packages to the archive. +This will involve choosing a dput host name. +That's probably your distro name, I<distro>. + +=back + +=head1 CONFIGURATION + +When you have all of the above set up, +you are ready to explain to dgit how to access your systems. + +dgit is configured via git's configuration system, +so this is done with git configuration. +See L<git-config(1)>. + +Below, each heading is one or more git config keys. +B<bold> is literal text and I<italics> +is things that vary. +In the descriptions of the effects of config settings, +we refer to the config values C<like this>. + +=over + +=item B<dgit-distro.>I<distro>B<.git-url>, B<.git-url-suffix> + +Specify the publicly accessible git URLs for your +dgit git server. The urls generated are +C<git-url>B</>I<package>C<git-url-suffix> + +The url should be stable, +and publicly accessible, +because its name is published in .dsc files. +(Note that if you make modified versions of packages from Debian, +the copyleft licences used for Free Software +often require you to permit your users, employees, and downstreams +to further distribute your modified source code.) + +=item B<dgit-distro.>I<distro>B</push.git-host> + +The domain name of your git server's ssh interface. + +=item B<dgit-distro.>I<distro>B</push.git-user-force> B<dgit-distro.>I<distro>B</push.username> + +Some git hosting systems expect everyone to +connect over ssh as the same user, often B<git>. +If this is the case, set C<git-user-force> to that user. + +If you have a normal git over ssh arrangement, +where people ssh as themselves, +leave C<git-user-force> unset. +If a user wishes to override the username +(for example, if their local username is not the same as on the server) +they can set C<username>. + +=item B<dgit-distro.>I<distro>B</push.git-url> + +Set this to the empty string. +This will arrange that push accesses to the ssh server will use +C</push.git-host>, etc. + +=item B<dgit-distro.>I<distro>B</push.git-proto> B<git+ssh://> + +=item C<dgit-distro.>I<distro>B</push.git-path> + +The path to your repositories. +dgit push will try to push to +C<git-proto>[C<git-user-force>|C<username>B<@>]C<git-path>B</>I<package>B<.git> + +=item B<dgit-distro.>I<distro>B<.git-check>, B<.git-check-suffix> + +dgit clone needs to be able to tell whether there is +yet a git repository for a particular package. + +If you always have a git repository for every package in your archive, +perhaps because you never use dput/dupload, and always dgit push, +Set C<git-check> to B<true>. + +Otherwise, set C<git-check> to a url prefix - ideally, https. +dgit clone will try to fetch +C<git-check>B</>I<package>C<git-check-suffix> +and expect to get either some successful fetch (it doesn't matter what) +or a file not found error (http 404 status code). +Other outcomes are fatal errors. + +If your git server runs cgit, +then you can set C<git-check> to the same as C<git-url>, +and C<git-check-suffix> to B</info/refs>. + +=item B<dgit-distro.>I<distro>B</push.git-check>, B</push.git-create> + +dgit push also needs to be able to check whether the repo exists. + +You can set both of these to B<ssh-cmd>, +which will use an ssh shell command to test repository existence. +Or leave them unset, and dgit push will use the readonly details. +If repositories are created automatically on push, somehow, +you can set C<git-create> to B<true>. + +=item B<dgit-distro.>I<distro>B<.upload-host> + +What I<host> value to pass to dput, to upload. + +This is a nickname, not the real host name. +You need to provide everyone who will push with an appropriate +dput configuration. +See L<dput.cf(5)>. + +A good nickname for your upload host is your distro name I<distro>. + +=item B<dgit-distro.>I<distro>B<.mirror> + +Set this to the url of your source package archive. +This is the same string as appears +in the 2nd field of each B<sources.list> entry. + +=item B<dgit-distro.>I<distro>B<.archive-query>, B<.archive-query-url> + +If you have a smallish distro, +set C<archive-query> to B<aptget:> (with a colon). + +If your distro is large +(eg, if it contains a substantial fraction of Debian) +then this will not be very efficient: +with this setting, +dgit often needs to download and update Sources files. + +For large distros, +it is better to implement the Debian archive ftpmaster API. +See L<https://api.ftp-master.debian.org/>, +and set C<archive-query> to B<ftpmasterapi:> (with a colon) +and C<archive-query-url> to your API base URL. +dgit uses these queries: +B<suites>, +B<dsc_in_suite/>I<isuite>B</>I<package> and +B<file_in_archive/>I<pat> +(so you need not implement anything else). + +Alternatively, +if your system supports the rmadison protocol, +you can set C<archive-query> to B<madison:>[I<madison-distro>]. +dgit will invoke B<rmadison> -uI<madison-distro>. + +=item B<dgit-suite.>I<suite>B<.distro> I<distro> + +Set this for every one of your suites. +You will have to update this when new suites are created. +If you forget, your users can explicitly specify B<-d> I<distro> +to dgit. + +=back + +=head1 TEMPLATE GIT REPOSITORY + +When dgit push is used for I<package> for the first time, +it must create a git repository on the git server. + +If C<git-create> is set to B<ssh-cmd>, +dgit will use the user's shell access to the server to +B<cp -a _template.git> I<package>B<.git>. +So you should create B<_template.git> with suitable contents. + +Note that the ssh rune invoked by dgit does not do any locking. +So if two people dgit push the same package at the same time, +there will be lossage. +Either don't do that, or set up B<dgit-repos-server>. + +=head1 SSH COMMANDS + +When a user who can push runs dgit, +dgit uses ssh to access the git server. + +To make ssh restricted command easier, +and for the benefit of dgit-repos-server, +dgit's ssh commands +each start with a parseable commentish rune. + +The ssh commands used by dgit are these: + +=over + +=item B<: dgit> I<distro> B<git-check> I<package> B<;>... + +Test whether I<package> has a git repo on the server already. +Should print B<0> or B<1> and a newline, +and exit status zero in either case. +The rest of the command, after B<;>, +is a shell implementation of this test. +Used when C<git-check> is set to B<ssh-cmd>. + +=item B<: dgit> I<distro> B<git-create> I<package> B<;>... + +Create the git repository for I<package> on the server. +See L</TEMPLATE GIT REPOSITORY>, above. +The rest of the command is an appropriate invocation of cd and cp. +Used when C<git-create> is set to B<ssh-cmd>. + +=item B<git-receive-pack>..., B<git-upload-pack>... + +dgit invokes git to access the repository; +git then runs these commands. +Note that dgit push will first do a git fetch over ssh, +so you must provide upload-pack as well as receive-pack. + +=back + +(There are also other ssh commands +which are historical or obscure.) + +=head1 SEE ALSO + +dgit(1) @@ -46,6 +46,7 @@ dgit-maint-debrebase(7) for maintainers: a pure-git rebasish workflow dgit-maint-merge(7) for maintainers: a pure-git merging workflow dgit-maint-gbp(7) for maintainers already using git-buildpackage dgit-sponsorship(7) for sponsors and sponsored contributors +dgit-downstream-dsc(7) setting up dgit push for a new distro .TE .LP See \fBdgit(7)\fP for detailed information about the data diff --git a/git-debrebase b/git-debrebase index f16a89c..f002225 100755 --- a/git-debrebase +++ b/git-debrebase @@ -41,6 +41,8 @@ our ($opt_defaultcmd_interactive); our $us = qw(git-debrebase); +$|=1; + sub badusage ($) { my ($m) = @_; print STDERR "bad usage: $m\n"; @@ -276,6 +278,11 @@ sub snags_maybe_bail () { } $snags_summarised = $snags_forced + $snags_tripped; } +sub snags_maybe_bail_early () { + # useful to bail out early without doing a lot of work; + # not a substitute for snags_maybe_bail. + snags_maybe_bail() if $snags_tripped && !$opt_force; +} sub any_snags () { return $snags_forced || $snags_tripped; } @@ -980,24 +987,27 @@ sub ffq_prev_branchinfo () { return gdr_ffq_prev_branchinfo($current); } -sub record_ffq_prev_deferred () { - # => ('status', "message") - # 'status' may be - # deferred message is undef +sub ffq_check ($;$$) { + # calls $ff and/or $notff zero or more times + # then returns either (status,message) where status is # exists # detached # weird-symref # notbranch - # if not ff from some branch we should be ff from, is an snag - # if "deferred", will have added something about that to - # @deferred_update_messages, and also maybe printed (already) - # some messages about ff checks + # or (undef,undef, $ffq_prev,$gdrlast) + # $ff and $notff are called like this: + # $ff->("message for stdout\n"); + # $notff->('snag-name', $message); + # normally $currentval should be HEAD + my ($currentval, $ff, $notff) =@_; + + $ff //= sub { print $_[0] or die $!; }; + $notff //= \&snag; + my ($status, $message, $current, $ffq_prev, $gdrlast) = ffq_prev_branchinfo(); return ($status, $message) unless $status eq 'branch'; - my $currentval = get_head(); - my $exists = git_get_ref $ffq_prev; return ('exists',"$ffq_prev already exists") if $exists; @@ -1024,14 +1034,14 @@ sub record_ffq_prev_deferred () { return unless length $lrval; if (is_fast_fwd $lrval, $currentval) { - print "OK, you are ahead of $lrref\n" or die $!; + $ff->("OK, you are ahead of $lrref\n"); $checked{$lrref} = 1; } elsif (is_fast_fwd $currentval, $lrval) { $checked{$lrref} = -1; - snag 'behind', "you are behind $lrref, divergence risk"; + $notff->('behind', "you are behind $lrref, divergence risk"); } else { $checked{$lrref} = -1; - snag 'diverged', "you have diverged from $lrref"; + $notff->('diverged', "you have diverged from $lrref"); } }; @@ -1056,6 +1066,25 @@ sub record_ffq_prev_deferred () { } elsif ($branch =~ m{^master$}) { $check->("refs/remotes/dgit/dgit/sid", 'remote dgit branch for sid'); } + return (undef, undef, $ffq_prev, $gdrlast); +} + +sub record_ffq_prev_deferred () { + # => ('status', "message") + # 'status' may be + # deferred message is undef + # exists + # detached + # weird-symref + # notbranch + # if not ff from some branch we should be ff from, is an snag + # if "deferred", will have added something about that to + # @deferred_update_messages, and also maybe printed (already) + # some messages about ff checks + my $currentval = get_head(); + + my ($status,$message, $ffq_prev,$gdrlast) = ffq_check $currentval; + return ($status,$message) if defined $status; snags_maybe_bail(); @@ -1474,7 +1503,7 @@ sub cmd_status () { sub cmd_stitch () { my $prose = 'stitch'; - GetOptions('prose=s', \$prose) or die badusage("bad options to stitch"); + GetOptions('prose=s', \$prose) or badusage("bad options to stitch"); badusage "no arguments allowed" if @ARGV; do_stitch $prose, 0; } @@ -1538,7 +1567,7 @@ sub make_patches ($) { sub cmd_make_patches () { my $opt_quiet_would_amend; GetOptions('quiet-would-amend!', \$opt_quiet_would_amend) - or die badusage("bad options to make-patches"); + or badusage("bad options to make-patches"); badusage "no arguments allowed" if @ARGV; my $old_head = get_head(); my $new = make_patches $old_head; @@ -1594,7 +1623,42 @@ sub cmd_convert_from_gbp () { "upstream ($upstream) contains debian/ directory"; } - snags_maybe_bail(); + my $previous_dgit_view = eval { + my @clogcmd = qw(dpkg-parsechangelog --format rfc822 -n2); + my ($lvsn, $suite); + parsechangelog_loop \@clogcmd, 'debian/changelog', sub { + my ($stz, $desc) = @_; + no warnings qw(exiting); + printdebug 'CHANGELOG ', Dumper($desc, $stz); + next unless $stz->{Date}; + next unless $stz->{Distribution} ne 'UNRELEASED'; + $lvsn = $stz->{Version}; + $suite = $stz->{Distribution}; + last; + }; + die "neither of the first two changelog entries are released\n" + unless defined $lvsn; + print "last finished-looking changelog entry: ($lvsn) $suite\n"; + my $mtag_pat = debiantag_maintview $lvsn, '*'; + my $mtag = cmdoutput @git, qw(describe --always --abbrev=0 --match), + $mtag_pat; + die "could not find suitable maintainer view tag $mtag_pat\n" + unless $mtag_pat =~ m{/}; + is_fast_fwd $mtag, 'HEAD' or + die "HEAD is not FF from maintainer tag $mtag!"; + my $dtag = "archive/$mtag"; + is_fast_fwd $mtag, $dtag or + die "dgit view tag $dtag is not FF from maintainer tag $mtag"; + print "will stitch in dgit view, $dtag\n"; + git_rev_parse $dtag; + }; + if (!$previous_dgit_view) { + $@ =~ s/^\n+//; + chomp $@; + print STDERR "cannot stitch in dgit view: $@\n"; + } + + snags_maybe_bail_early(); my $work; @@ -1623,8 +1687,17 @@ sub cmd_convert_from_gbp () { runcmd @git, qw(reset --quiet --hard patch-queue/gdr-internal); runcmd @git, qw(rebase --quiet --onto), $work, qw(gdr-internal); $work = git_rev_parse 'HEAD'; + + if ($previous_dgit_view) { + $work = make_commit [$work, $previous_dgit_view], [ + 'git-debrebase import: declare ff from dgit archive view', + '[git-debrebase pseudomerge: import-from-gbp]', + ]; + } }; + ffq_check $work; + snags_maybe_bail(); update_head_checkout $old_head, $work, 'convert-from-gbp'; } @@ -1719,7 +1792,7 @@ GetOptions("D+" => \$debuglevel, # approach. '-i=s{0,}' does not work with bundling. push @$opt_defaultcmd_interactive, @ARGV; @ARGV=(); - }) or die badusage "bad options\n"; + }) or badusage "bad options\n"; initdebug('git-debrebase '); enabledebug if $debuglevel; diff --git a/git-debrebase.1.pod b/git-debrebase.1.pod index 4d1a673..273ef47 100644 --- a/git-debrebase.1.pod +++ b/git-debrebase.1.pod @@ -257,9 +257,18 @@ This check exists to detect certain likely user errors, but if this situation is true and expected, forcing it is fine. +git-debrebase will try to look for the dgit archive view +of the most recent release, +and if it finds it will make a pseduomerge so that +your new git-debrebase view is appropriately fast forward. + The result is a well-formed git-debrebase interchange branch. The result is also fast-forward from the gbp branch. +It is a snag if the new branch looks like it will have diverged, +just as for a laundering/unstitching call to git-debrebase; +See L</Establish the current branch's ffq-prev>, below. + Note that it is dangerous not to know whether you are dealing with a gbp patches-unappled branch containing quilt patches, or a git-debrebase interchange branch. diff --git a/git-debrebase.5.pod b/git-debrebase.5.pod index e445c0e..52fb60b 100644 --- a/git-debrebase.5.pod +++ b/git-debrebase.5.pod @@ -15,6 +15,8 @@ as a series of individual git commits, which can worked on with rebase, and also shared. +=head2 DISCUSSION + git-debrebase is designed to work well with dgit. git-debrebase can also be used in workflows without source packages, for example to work on Debian-format packages outside or alongside Debian. @@ -27,6 +29,11 @@ provided by your upstream. However, use of git-debrebase in Debian does not make anything harder for derivatives, and it can make some things easier. +When using gitk on branches managed by git-debrebase, +B<gitk --date-order>, B<gitk --first-parent> +and B<gitk -- :.> (or B<gitk .>) +produce more useful output than the default. + =head1 TERMINOLOGY =over diff --git a/infra/dgit-mirror-rsync b/infra/dgit-mirror-rsync index 9346489..2d912c8 100755 --- a/infra/dgit-mirror-rsync +++ b/infra/dgit-mirror-rsync @@ -54,6 +54,7 @@ rsync=(rsync -rltH --safe-links --delete) hooktimeout=30 rsynctimeout=900 rsyncssh='ssh -o batchmode=yes' +mirror_gc_cmd='git gc --auto' . $distrodir/mirror-settings @@ -74,6 +75,12 @@ case "$remoterepos" in esac actually () { + if [ "x$mirror_gc_cmd" != x ]; then + ( + cd "$repos/$package.git" + $mirror_gc_cmd + ) + fi "${rsync[@]}" \ --timeout=$rsynctimeout \ -e "$rsyncssh" \ @@ -17,6 +17,8 @@ funcs: ${FUNCNAME[*]} lines: ${BASH_LINENO[*]} files: ${BASH_SOURCE[*]} END + t-save-artifacts + exit 16 } @@ -409,9 +411,18 @@ t-fscks () { t-ok () { : '========================================' t-fscks + t-save-artifacts echo ok. } +t-save-artifacts () { + artifacts="$AUTOPKGTEST_ARTIFACTS" + if [ x"$artifacts" = x ]; then return; fi + if [ x"tmp" = x ]; then return; fi + GZIP=-1v tar -C "$tmp" -zc -f "$artifacts/${0##*/}.tar.gz" \ + --exclude=\*.tar . +} + t-rm-dput-dropping () { rm -f $tmp/${p}_${v}_*.upload } @@ -1006,6 +1017,16 @@ t-commit () { revision=$(( ${revision-0} + 1 )) } +t-dch-r-rune () { + local cmd="$1"; shift + local suite=${1-unstable} + $cmd -r -D "$suite" '' +} + +t-dch-commit-r () { + t-dch-r-rune t-dch-commit "$@" +} + t-dch-commit () { faketime @"${GIT_AUTHOR_DATE% *}" dch "$@" git commit -m "dch $*" debian/changelog diff --git a/tests/lib-gdr b/tests/lib-gdr index 22ea6d1..95d2330 100644 --- a/tests/lib-gdr +++ b/tests/lib-gdr @@ -222,7 +222,7 @@ t-nmu-upload-2 () { } t-nmu-upload-3 () { - t-dch-commit -r sid + t-dch-commit-r t-dgit -wgf build-source diff --git a/tests/playtree-save-refs b/tests/playtree-save-refs new file mode 100755 index 0000000..7841367 --- /dev/null +++ b/tests/playtree-save-refs @@ -0,0 +1,9 @@ +#!/bin/bash +set -e +set -o pipefail +find -name .git -type d -exec sh -xec ' + cd "$1" + git for-each-ref \ + --format='\''update refs/dgit-test-keep/g%(objectname) %(objectname)'\'' \ +' x "{}" \; | \ + sort -u | git update-ref --stdin diff --git a/tests/setup/examplegit b/tests/setup/examplegit index 112e27a..48864d4 100755 --- a/tests/setup/examplegit +++ b/tests/setup/examplegit @@ -49,5 +49,3 @@ t-commit Sid push-to master sid t-setup-done 'p v suitespecs majorv revision' "aq git mirror $p" - -t-ok diff --git a/tests/setup/gdr-convert-gbp b/tests/setup/gdr-convert-gbp index e523fd3..6ec374d 100755 --- a/tests/setup/gdr-convert-gbp +++ b/tests/setup/gdr-convert-gbp @@ -64,7 +64,7 @@ gbp import-orig --upstream-version=2.0 ../$ust not-gdr-processable t-dch-commit -v 2.0-1 -m 'new upstream (did gbp import-orig)' -t-dch-commit -r sid +t-dch-commit-r $ifarchive t-archive-none $p $ifarchive t-git-none @@ -84,9 +84,9 @@ t-git-debrebase -fupstream-has-debian convert-from-gbp v=2.0-2 t-dch-commit -v $v -m 'switch to git-debrebase, no other changes' -t-dch-commit -r sid +t-dch-commit-r -$ifarchive t-dgit -wgf push-source --new --overwrite +$ifarchive t-dgit -wgf push-source --new git push cd .. @@ -99,5 +99,3 @@ t-setup-done '' "$(echo $p*) salsa $($ifarchive echo git mirror aq)" ' p=example t-git-next-date ' - -t-ok diff --git a/tests/setup/gnupg b/tests/setup/gnupg index 7a164ef..56e4e66 100755 --- a/tests/setup/gnupg +++ b/tests/setup/gnupg @@ -30,5 +30,3 @@ gpg --list-secret t-setup-done 'DGIT_TEST_GNUPG_GLOBAL_LOCK DGIT_TEST_GNUPG_LOG' \ 'gnupg' "$setup" - -t-ok diff --git a/tests/tests/gdr-diverge-nmu-dgit b/tests/tests/gdr-diverge-nmu-dgit index 4b5907a..1d136a9 100755 --- a/tests/tests/gdr-diverge-nmu-dgit +++ b/tests/tests/gdr-diverge-nmu-dgit @@ -28,7 +28,7 @@ t-git-next-date v=2.0-2+nmu1 t-nmu-commit-an-upstream-change t-dch-commit -v$v -m finalise -t-dch-commit -r sid +t-dch-commit-r t-dgit -wgf push-source diff --git a/tests/tests/gdr-edits b/tests/tests/gdr-edits index 6c77184..52c083b 100755 --- a/tests/tests/gdr-edits +++ b/tests/tests/gdr-edits @@ -16,7 +16,7 @@ cat ../anal.1 t-some-changes edits -t-dch-commit -r sid +t-dch-commit-r git tag t.before diff --git a/tests/tests/gdr-import-dgit b/tests/tests/gdr-import-dgit index 19918d8..c18c097 100755 --- a/tests/tests/gdr-import-dgit +++ b/tests/tests/gdr-import-dgit @@ -40,7 +40,7 @@ nmu-fold v=2.0-3 t-dch-commit -v $v -m "incorporate nmu" -t-dch-commit -r sid +t-dch-commit-r t-dgit -wgf push-source : 'now test a new upstream' |