Merge branch '3.x' into merge_3.x_into_master

Resolved Conflicts:
	sphinx/texinputs/sphinx.sty

Some hunks needed to go to sphinx/texinputs/sphinxlatexliterals.py,
others already belonged to split files.

3 files changed, 255 insertions, 5 deletions

diff --git a/sphinx/texinputs/sphinx.sty b/sphinx/texinputs/sphinx.sty
index 6b7e5dcce..9edde8498 100644
--- a/sphinx/texinputs/sphinx.sty
+++ b/sphinx/texinputs/sphinx.sty
@@ -92,6 +92,9 @@
 % verbatim
 \DeclareBoolOption[true]{verbatimwithframe}
 \DeclareBoolOption[true]{verbatimwrapslines}
+\DeclareBoolOption[false]{verbatimforcewraps}
+\DeclareStringOption[3]{verbatimmaxoverfull}
+\DeclareStringOption[100]{verbatimmaxunderfull}
 \DeclareBoolOption[true]{verbatimhintsturnover}
 \DeclareBoolOption[true]{inlineliteralwraps}
 \DeclareStringOption[t]{literalblockcappos}
@@ -298,6 +301,28 @@
 
 %% FOOTNOTES
 %
+% Support scopes for footnote numbering
+\newcounter{sphinxscope}
+\newcommand{\sphinxstepscope}{\stepcounter{sphinxscope}}
+% Explictly numbered footnotes may be referred to, and for this to be
+% clickable we need to have only one target. So we will step this at each
+% explicit footnote and let \thesphinxscope take it into account
+\newcounter{sphinxexplicit}
+\newcommand{\sphinxstepexplicit}{\stepcounter{sphinxexplicit}}
+% Some babel/polyglossia languages fiddle with \@arabic, so let's be extra
+% cautious and redefine \thesphinxscope with \number not \@arabic.
+% Memo: we expect some subtle redefinition of \thesphinxscope to be a part of page
+% scoping for footnotes, when we shall implement it.
+\renewcommand{\thesphinxscope}{\number\value{sphinxscope}.\number\value{sphinxexplicit}}
+\newcommand\sphinxthefootnotemark[2]{%
+   % this is used to make reference to an explicitly numbered footnote not on same page
+   % #1=label of footnote text, #2=page number where footnote text was printed
+   \ifdefined\pagename
+      \pagename\space#2, % <- space
+    \else
+      p. #2, % <- space
+   \fi #1% no space
+}
 % support large numbered footnotes in minipage; but this is now obsolete
 % from systematic use of savenotes environment around minipages
 \def\thempfootnote{\arabic{mpfootnote}}
diff --git a/sphinx/texinputs/sphinxlatexliterals.sty b/sphinx/texinputs/sphinxlatexliterals.sty
index bda032bca..3aed425d7 100644
--- a/sphinx/texinputs/sphinxlatexliterals.sty
+++ b/sphinx/texinputs/sphinxlatexliterals.sty
@@ -268,13 +268,188 @@
   % no need to restore \fboxsep here, as this ends up in a \hbox from fancyvrb
 }%
 % \sphinxVerbatimFormatLine will be set locally to one of those two:
-\newcommand\sphinxVerbatimFormatLineWrap[1]{%
-    \hsize\linewidth
+\newcommand\sphinxVerbatimFormatLineWrap{%
+  \hsize\linewidth
+  \ifspx@opt@verbatimforcewraps
+       \expandafter\spx@verb@FormatLineForceWrap
+  \else\expandafter\spx@verb@FormatLineWrap
+  \fi
+}%
+\newcommand\sphinxVerbatimFormatLineNoWrap[1]{\hb@xt@\linewidth{\strut #1\hss}}%
+\long\def\spx@verb@FormatLineWrap#1{%
     \vtop{\raggedright\hyphenpenalty\z@\exhyphenpenalty\z@
           \doublehyphendemerits\z@\finalhyphendemerits\z@
           \strut #1\strut}%
 }%
-\newcommand\sphinxVerbatimFormatLineNoWrap[1]{\hb@xt@\linewidth{\strut #1\hss}}%
+%
+% The normal line wrapping allows breaks at spaces and ascii non
+% letters, non digits. The \raggedright above means there will be
+% an overfilled line only if some non-breakable "word" was
+% encountered, which is longer than a line (it is moved always to
+% be on its own on a new line).
+% 
+% The "forced" line wrapping will parse the tokens to add potential
+% breakpoints at each character. As some strings are highlighted,
+% we have to apply the highlighting character per character, which
+% requires to manipulate the output of the Pygments LaTeXFormatter.
+%
+% Doing this at latex level is complicated. The contents should
+% be as expected: i.e. some active characters from
+% \sphinxbreaksviaactive, some Pygments character escapes such as
+% \PYGZdl{}, and the highlighting \PYG macro with always 2
+% arguments. No other macros should be there, except perhaps
+% zero-parameter macros. In particular:
+% - the texcomments Pygments option must be set to False
+%
+% With pdflatex, Unicode input gives multi-bytes characters
+% where the first byte is active. We support the "utf8" macros
+% only. "utf8x" is not supported.
+%
+% The highlighting macro \PYG will be applied character per
+% character. Highlighting via a colored background gives thus a
+% chain of small colored boxes which may cause some artefact in
+% some pdf viewers. Can't do anything here if we do want the line
+% break to be possible.
+%
+% First a measurement step is done of what would the standard line
+% wrapping give (i.e line breaks only at spaces and non-letter,
+% non-digit ascii characters), cf TeX by Topic for the basic
+% dissecting technique: TeX unfortunately when building a vertical
+% box does not store in an accessible way what was the maximal
+% line-width during paragraph building.
+%
+% If the max width exceeds the linewidth by more than verbatimmaxoverfull
+% character widths, or if the min width plus verbatimmaxunderfull character
+% widths is inferior to linewidth, then we apply the "force wrapping" with
+% potential line break at each character, else we don't.
+\long\def\spx@verb@FormatLineForceWrap#1{%
+    % \spx@image@box is a scratch box register that we can use here
+    \global\let\spx@verb@maxwidth\z@
+    \global\let\spx@verb@minwidth\linewidth
+    \setbox\spx@image@box
+    \vtop{\raggedright\hyphenpenalty\z@\exhyphenpenalty\z@
+          \doublehyphendemerits\z@\finalhyphendemerits\z@
+          \strut #1\strut\@@par
+          \spx@verb@getwidths}%
+    \ifdim\spx@verb@maxwidth>
+          \dimexpr\linewidth+\spx@opt@verbatimmaxoverfull\fontcharwd\font`X \relax
+      \spx@verb@FormatLineWrap{\spx@verb@wrapPYG #1\spx@verb@wrapPYG}%
+    \else
+      \ifdim\spx@verb@minwidth<
+            \dimexpr\linewidth-\spx@opt@verbatimmaxunderfull\fontcharwd\font`X \relax
+        \spx@verb@FormatLineWrap{\spx@verb@wrapPYG #1\spx@verb@wrapPYG}%
+      \else
+        \spx@verb@FormatLineWrap{#1}%
+    \fi\fi
+}%
+% auxiliary paragraph dissector to get max and min widths
+\newbox\spx@scratchbox
+\def\spx@verb@getwidths {%
+    \unskip\unpenalty
+    \setbox\spx@scratchbox\lastbox
+    \ifvoid\spx@scratchbox
+    \else
+       \setbox\spx@scratchbox\hbox{\unhbox\spx@scratchbox}%
+       \ifdim\spx@verb@maxwidth<\wd\spx@scratchbox
+          \xdef\spx@verb@maxwidth{\number\wd\spx@scratchbox sp}%
+       \fi
+       \ifdim\spx@verb@minwidth>\wd\spx@scratchbox
+          \xdef\spx@verb@minwidth{\number\wd\spx@scratchbox sp}%
+       \fi
+       \expandafter\spx@verb@getwidths
+    \fi
+}%
+% auxiliary macros to implement "cut long line even in middle of word"
+\catcode`Z=3 % safe delimiter
+\def\spx@verb@wrapPYG{%
+    \futurelet\spx@nexttoken\spx@verb@wrapPYG@i
+}%
+\def\spx@verb@wrapPYG@i{%
+    \ifx\spx@nexttoken\spx@verb@wrapPYG\let\next=\@gobble\else
+    \ifx\spx@nexttoken\PYG\let\next=\spx@verb@wrapPYG@PYG@onebyone\else
+      \discretionary{}{\sphinxafterbreak}{}%
+      \let\next\spx@verb@wrapPYG@ii
+    \fi\fi
+    \next
+}%
+% Let's recognize active characters. We don't support utf8x only utf8.
+% And here #1 should not have picked up (non empty) braced contents
+\long\def\spx@verb@wrapPYG@ii#1{%
+    \ifcat\noexpand~\noexpand#1\relax% active character
+      \expandafter\spx@verb@wrapPYG@active
+    \else % non-active character, control sequence such as \PYGZdl, or empty
+      \expandafter\spx@verb@wrapPYG@one
+    \fi {#1}%
+}%
+\long\def\spx@verb@wrapPYG@active#1{%
+% Let's hope expansion of active character does not really require arguments,
+% as we certainly don't want to go into expanding upfront token stream anyway.
+    \expandafter\spx@verb@wrapPYG@iii#1{}{}{}{}{}{}{}{}{}Z#1%
+}%
+\long\def\spx@verb@wrapPYG@iii#1#2Z{%
+    \ifx\UTFviii@four@octets#1\let\next=\spx@verb@wrapPYG@four\else
+    \ifx\UTFviii@three@octets#1\let\next=\spx@verb@wrapPYG@three\else
+    \ifx\UTFviii@two@octets#1\let\next=\spx@verb@wrapPYG@two\else
+    \let\next=\spx@verb@wrapPYG@one
+    \fi\fi\fi
+    \next
+}%
+\long\def\spx@verb@wrapPYG@one   #1{#1\futurelet\spx@nexttoken\spx@verb@wrapPYG@i}%
+\long\def\spx@verb@wrapPYG@two   #1#2{#1#2\futurelet\spx@nexttoken\spx@verb@wrapPYG@i}%
+\long\def\spx@verb@wrapPYG@three #1#2#3{#1#2#3\futurelet\spx@nexttoken\spx@verb@wrapPYG@i}%
+\long\def\spx@verb@wrapPYG@four  #1#2#3#4{#1#2#3#4\futurelet\spx@nexttoken\spx@verb@wrapPYG@i}%
+% Replace \PYG by itself applied one character at a time! This way breakpoints
+% can be inserted.
+\def\spx@verb@wrapPYG@PYG@onebyone#1#2#3{% #1 = \PYG, #2 = highlight spec, #3 = tokens
+    \def\spx@verb@wrapPYG@PYG@spec{{#2}}%
+    \futurelet\spx@nexttoken\spx@verb@wrapPYG@PYG@i#3Z%
+}%
+\def\spx@verb@wrapPYG@PYG@i{%
+    \ifx\spx@nexttokenZ\let\next=\spx@verb@wrapPYG@PYG@done\else
+      \discretionary{}{\sphinxafterbreak}{}%
+      \let\next\spx@verb@wrapPYG@PYG@ii
+    \fi
+    \next
+}%
+\def\spx@verb@wrapPYG@PYG@doneZ{\futurelet\spx@nexttoken\spx@verb@wrapPYG@i}%
+\long\def\spx@verb@wrapPYG@PYG@ii#1{%
+    \ifcat\noexpand~\noexpand#1\relax% active character
+      \expandafter\spx@verb@wrapPYG@PYG@active
+    \else % non-active character, control sequence such as \PYGZdl, or empty
+      \expandafter\spx@verb@wrapPYG@PYG@one
+    \fi {#1}%
+}%
+\long\def\spx@verb@wrapPYG@PYG@active#1{%
+% Let's hope expansion of active character does not really require arguments,
+% as we certainly don't want to go into expanding upfront token stream anyway.
+    \expandafter\spx@verb@wrapPYG@PYG@iii#1{}{}{}{}{}{}{}{}{}Z#1%
+}%
+\long\def\spx@verb@wrapPYG@PYG@iii#1#2Z{%
+    \ifx\UTFviii@four@octets#1\let\next=\spx@verb@wrapPYG@PYG@four\else
+    \ifx\UTFviii@three@octets#1\let\next=\spx@verb@wrapPYG@PYG@three\else
+    \ifx\UTFviii@two@octets#1\let\next=\spx@verb@wrapPYG@PYG@two\else
+    \let\next=\spx@verb@wrapPYG@PYG@one
+    \fi\fi\fi
+    \next
+}%
+\long\def\spx@verb@wrapPYG@PYG@one#1{%
+    \expandafter\PYG\spx@verb@wrapPYG@PYG@spec{#1}%
+    \futurelet\spx@nexttoken\spx@verb@wrapPYG@PYG@i
+}%
+\long\def\spx@verb@wrapPYG@PYG@two#1#2{%
+    \expandafter\PYG\spx@verb@wrapPYG@PYG@spec{#1#2}%
+    \futurelet\spx@nexttoken\spx@verb@wrapPYG@PYG@i
+}%
+\long\def\spx@verb@wrapPYG@PYG@three#1#2#3{%
+    \expandafter\PYG\spx@verb@wrapPYG@PYG@spec{#1#2#3}%
+    \futurelet\spx@nexttoken\spx@verb@wrapPYG@PYG@i
+}%
+\long\def\spx@verb@wrapPYG@PYG@four#1#2#3#4{%
+    \expandafter\PYG\spx@verb@wrapPYG@PYG@spec{#1#2#3#4}%
+    \futurelet\spx@nexttoken\spx@verb@wrapPYG@PYG@i
+}%
+\catcode`Z 11%
+%
 \g@addto@macro\FV@SetupFont{%
     \sbox\sphinxcontinuationbox {\spx@opt@verbatimcontinued}%
     \sbox\sphinxvisiblespacebox {\spx@opt@verbatimvisiblespace}%
diff --git a/sphinx/texinputs/sphinxpackagefootnote.sty b/sphinx/texinputs/sphinxpackagefootnote.sty
index 529d24f33..a6071cf10 100644
--- a/sphinx/texinputs/sphinxpackagefootnote.sty
+++ b/sphinx/texinputs/sphinxpackagefootnote.sty
@@ -331,8 +331,58 @@
 % some extras for Sphinx :
 % \sphinxfootnotemark: usable in section titles and silently removed from TOCs.
 \def\sphinxfootnotemark [#1]%
-   {\ifx\thepage\relax\else\protect\spx@opt@BeforeFootnote
-                             \protect\footnotemark[#1]\fi}%
+   {\ifx\thepage\relax\else\sphinxfootref{#1}\fi}%
+% \sphinxfootref:
+% - \spx@opt@BeforeFootnote is from BeforeFootnote sphinxsetup option
+% - \ref:
+%     the latex.py writer inserts a \phantomsection\label{<scope>.<num>}
+%     whenever
+%     - the footnote was explicitly numbered in sources,
+%     - or it was in restrained context and is rendered using footnotetext
+%
+%     These are the two types of footnotes that \sphinxfootnotemark must
+%     handle. But for explicitly numbered footnotes the same number
+%     can be found in document. So a secondary part in  <scope> is updated
+%     at each novel such footnote to know what is the target from then on
+%     for \sphinxfootnotemark and already encountered [1], or [2],...
+%
+% LaTeX package varioref is not supported by hyperref (from its doc: "There
+% are too many problems with varioref. Nobody has time to sort them out.
+% Therefore this package is now unsupported.") So we will simply use our own
+% macros to access the page number of footnote text and decide whether to print
+% it. \pagename is internationalized by latex-babel.
+\def\spx@thefnmark#1#2{%
+  % #1=label for reference, #2=page where footnote was printed
+  \ifx\spx@tempa\spx@tempb
+     % same page
+     #1%
+  \else
+    \sphinxthefootnotemark{#1}{#2}%
+  \fi
+}%
+\def\sphinxfootref@get #1#2#3#4#5\relax{%
+    \def\sphinxfootref@label{#1}%
+    \def\sphinxfootref@page {#2}%
+    \def\sphinxfootref@Href {#4}%
+}%
+\protected\def\sphinxfootref#1{% #1 always explicit number in Sphinx usage
+  \spx@opt@BeforeFootnote
+  \ltx@ifundefined{r@\thesphinxscope.#1}%
+    {\gdef\@thefnmark{?}\H@@footnotemark}%
+    {\expandafter\expandafter\expandafter\sphinxfootref@get
+                 \csname r@\thesphinxscope.#1\endcsname\relax
+     \edef\spx@tempa{\thepage}\edef\spx@tempb{\sphinxfootref@page}%
+     \protected@xdef\@thefnmark{\spx@thefnmark{\sphinxfootref@label}{\sphinxfootref@page}}%
+     \let\spx@@makefnmark\@makefnmark
+     \def\@makefnmark{%
+       \hyper@linkstart{link}{\sphinxfootref@Href}%
+       \spx@@makefnmark
+       \hyper@linkend
+     }%
+     \H@@footnotemark
+     \let\@makefnmark\spx@@makefnmark
+    }%
+}%
 \AtBeginDocument{%
    % let hyperref less complain
    \pdfstringdefDisableCommands{\def\sphinxfootnotemark [#1]{}}%