Merge branch '3.x' into latex_hyperlinked_caption_footnotes_on_3.x

1 files changed, 182 insertions, 4 deletions

diff --git a/sphinx/texinputs/sphinx.sty b/sphinx/texinputs/sphinx.sty
index 57ecdd441..556302930 100644
--- a/sphinx/texinputs/sphinx.sty
+++ b/sphinx/texinputs/sphinx.sty
@@ -334,6 +334,9 @@
 % verbatim
 \DeclareBoolOption[true]{verbatimwithframe}
 \DeclareBoolOption[true]{verbatimwrapslines}
+\DeclareBoolOption[false]{verbatimforcewraps}
+\DeclareStringOption[3]{verbatimmaxoverfull}
+\DeclareStringOption[100]{verbatimmaxunderfull}
 \DeclareBoolOption[true]{verbatimhintsturnover}
 \DeclareBoolOption[true]{inlineliteralwraps}
 \DeclareStringOption[t]{literalblockcappos}
@@ -414,7 +417,7 @@
 \DisableKeyvalOption{sphinx}{mathnumfig}
 % To allow hyphenation of first word in narrow contexts; no option,
 % customization to be done via 'preamble' key
-\newcommand*\sphinxAtStartPar{\nobreak\hskip\z@skip}
+\newcommand*\sphinxAtStartPar{\hskip\z@skip}
 % No need for the \hspace{0pt} trick (\hskip\z@skip) with luatex
 \ifdefined\directlua\let\sphinxAtStartPar\@empty\fi
 % user interface: options can be changed midway in a document!
@@ -1193,13 +1196,188 @@
   % no need to restore \fboxsep here, as this ends up in a \hbox from fancyvrb
 }%
 % \sphinxVerbatimFormatLine will be set locally to one of those two:
-\newcommand\sphinxVerbatimFormatLineWrap[1]{%
-    \hsize\linewidth
+\newcommand\sphinxVerbatimFormatLineWrap{%
+  \hsize\linewidth
+  \ifspx@opt@verbatimforcewraps
+       \expandafter\spx@verb@FormatLineForceWrap
+  \else\expandafter\spx@verb@FormatLineWrap
+  \fi
+}%
+\newcommand\sphinxVerbatimFormatLineNoWrap[1]{\hb@xt@\linewidth{\strut #1\hss}}%
+\long\def\spx@verb@FormatLineWrap#1{%
     \vtop{\raggedright\hyphenpenalty\z@\exhyphenpenalty\z@
           \doublehyphendemerits\z@\finalhyphendemerits\z@
           \strut #1\strut}%
 }%
-\newcommand\sphinxVerbatimFormatLineNoWrap[1]{\hb@xt@\linewidth{\strut #1\hss}}%
+%
+% The normal line wrapping allows breaks at spaces and ascii non
+% letters, non digits. The \raggedright above means there will be
+% an overfilled line only if some non-breakable "word" was
+% encountered, which is longer than a line (it is moved always to
+% be on its own on a new line).
+% 
+% The "forced" line wrapping will parse the tokens to add potential
+% breakpoints at each character. As some strings are highlighted,
+% we have to apply the highlighting character per character, which
+% requires to manipulate the output of the Pygments LaTeXFormatter.
+%
+% Doing this at latex level is complicated. The contents should
+% be as expected: i.e. some active characters from
+% \sphinxbreaksviaactive, some Pygments character escapes such as
+% \PYGZdl{}, and the highlighting \PYG macro with always 2
+% arguments. No other macros should be there, except perhaps
+% zero-parameter macros. In particular:
+% - the texcomments Pygments option must be set to False
+%
+% With pdflatex, Unicode input gives multi-bytes characters
+% where the first byte is active. We support the "utf8" macros
+% only. "utf8x" is not supported.
+%
+% The highlighting macro \PYG will be applied character per
+% character. Highlighting via a colored background gives thus a
+% chain of small colored boxes which may cause some artefact in
+% some pdf viewers. Can't do anything here if we do want the line
+% break to be possible.
+%
+% First a measurement step is done of what would the standard line
+% wrapping give (i.e line breaks only at spaces and non-letter,
+% non-digit ascii characters), cf TeX by Topic for the basic
+% dissecting technique: TeX unfortunately when building a vertical
+% box does not store in an accessible way what was the maximal
+% line-width during paragraph building.
+%
+% If the max width exceeds the linewidth by more than verbatimmaxoverfull
+% character widths, or if the min width plus verbatimmaxunderfull character
+% widths is inferior to linewidth, then we apply the "force wrapping" with
+% potential line break at each character, else we don't.
+\long\def\spx@verb@FormatLineForceWrap#1{%
+    % \spx@image@box is a scratch box register that we can use here
+    \global\let\spx@verb@maxwidth\z@
+    \global\let\spx@verb@minwidth\linewidth
+    \setbox\spx@image@box
+    \vtop{\raggedright\hyphenpenalty\z@\exhyphenpenalty\z@
+          \doublehyphendemerits\z@\finalhyphendemerits\z@
+          \strut #1\strut\@@par
+          \spx@verb@getwidths}%
+    \ifdim\spx@verb@maxwidth>
+          \dimexpr\linewidth+\spx@opt@verbatimmaxoverfull\fontcharwd\font`X \relax
+      \spx@verb@FormatLineWrap{\spx@verb@wrapPYG #1\spx@verb@wrapPYG}%
+    \else
+      \ifdim\spx@verb@minwidth<
+            \dimexpr\linewidth-\spx@opt@verbatimmaxunderfull\fontcharwd\font`X \relax
+        \spx@verb@FormatLineWrap{\spx@verb@wrapPYG #1\spx@verb@wrapPYG}%
+      \else
+        \spx@verb@FormatLineWrap{#1}%
+    \fi\fi
+}%
+% auxiliary paragraph dissector to get max and min widths
+\newbox\spx@scratchbox
+\def\spx@verb@getwidths {%
+    \unskip\unpenalty
+    \setbox\spx@scratchbox\lastbox
+    \ifvoid\spx@scratchbox
+    \else
+       \setbox\spx@scratchbox\hbox{\unhbox\spx@scratchbox}%
+       \ifdim\spx@verb@maxwidth<\wd\spx@scratchbox
+          \xdef\spx@verb@maxwidth{\number\wd\spx@scratchbox sp}%
+       \fi
+       \ifdim\spx@verb@minwidth>\wd\spx@scratchbox
+          \xdef\spx@verb@minwidth{\number\wd\spx@scratchbox sp}%
+       \fi
+       \expandafter\spx@verb@getwidths
+    \fi
+}%
+% auxiliary macros to implement "cut long line even in middle of word"
+\catcode`Z=3 % safe delimiter
+\def\spx@verb@wrapPYG{%
+    \futurelet\spx@nexttoken\spx@verb@wrapPYG@i
+}%
+\def\spx@verb@wrapPYG@i{%
+    \ifx\spx@nexttoken\spx@verb@wrapPYG\let\next=\@gobble\else
+    \ifx\spx@nexttoken\PYG\let\next=\spx@verb@wrapPYG@PYG@onebyone\else
+      \discretionary{}{\sphinxafterbreak}{}%
+      \let\next\spx@verb@wrapPYG@ii
+    \fi\fi
+    \next
+}%
+% Let's recognize active characters. We don't support utf8x only utf8.
+% And here #1 should not have picked up (non empty) braced contents
+\long\def\spx@verb@wrapPYG@ii#1{%
+    \ifcat\noexpand~\noexpand#1\relax% active character
+      \expandafter\spx@verb@wrapPYG@active
+    \else % non-active character, control sequence such as \PYGZdl, or empty
+      \expandafter\spx@verb@wrapPYG@one
+    \fi {#1}%
+}%
+\long\def\spx@verb@wrapPYG@active#1{%
+% Let's hope expansion of active character does not really require arguments,
+% as we certainly don't want to go into expanding upfront token stream anyway.
+    \expandafter\spx@verb@wrapPYG@iii#1{}{}{}{}{}{}{}{}{}Z#1%
+}%
+\long\def\spx@verb@wrapPYG@iii#1#2Z{%
+    \ifx\UTFviii@four@octets#1\let\next=\spx@verb@wrapPYG@four\else
+    \ifx\UTFviii@three@octets#1\let\next=\spx@verb@wrapPYG@three\else
+    \ifx\UTFviii@two@octets#1\let\next=\spx@verb@wrapPYG@two\else
+    \let\next=\spx@verb@wrapPYG@one
+    \fi\fi\fi
+    \next
+}%
+\long\def\spx@verb@wrapPYG@one   #1{#1\futurelet\spx@nexttoken\spx@verb@wrapPYG@i}%
+\long\def\spx@verb@wrapPYG@two   #1#2{#1#2\futurelet\spx@nexttoken\spx@verb@wrapPYG@i}%
+\long\def\spx@verb@wrapPYG@three #1#2#3{#1#2#3\futurelet\spx@nexttoken\spx@verb@wrapPYG@i}%
+\long\def\spx@verb@wrapPYG@four  #1#2#3#4{#1#2#3#4\futurelet\spx@nexttoken\spx@verb@wrapPYG@i}%
+% Replace \PYG by itself applied one character at a time! This way breakpoints
+% can be inserted.
+\def\spx@verb@wrapPYG@PYG@onebyone#1#2#3{% #1 = \PYG, #2 = highlight spec, #3 = tokens
+    \def\spx@verb@wrapPYG@PYG@spec{{#2}}%
+    \futurelet\spx@nexttoken\spx@verb@wrapPYG@PYG@i#3Z%
+}%
+\def\spx@verb@wrapPYG@PYG@i{%
+    \ifx\spx@nexttokenZ\let\next=\spx@verb@wrapPYG@PYG@done\else
+      \discretionary{}{\sphinxafterbreak}{}%
+      \let\next\spx@verb@wrapPYG@PYG@ii
+    \fi
+    \next
+}%
+\def\spx@verb@wrapPYG@PYG@doneZ{\futurelet\spx@nexttoken\spx@verb@wrapPYG@i}%
+\long\def\spx@verb@wrapPYG@PYG@ii#1{%
+    \ifcat\noexpand~\noexpand#1\relax% active character
+      \expandafter\spx@verb@wrapPYG@PYG@active
+    \else % non-active character, control sequence such as \PYGZdl, or empty
+      \expandafter\spx@verb@wrapPYG@PYG@one
+    \fi {#1}%
+}%
+\long\def\spx@verb@wrapPYG@PYG@active#1{%
+% Let's hope expansion of active character does not really require arguments,
+% as we certainly don't want to go into expanding upfront token stream anyway.
+    \expandafter\spx@verb@wrapPYG@PYG@iii#1{}{}{}{}{}{}{}{}{}Z#1%
+}%
+\long\def\spx@verb@wrapPYG@PYG@iii#1#2Z{%
+    \ifx\UTFviii@four@octets#1\let\next=\spx@verb@wrapPYG@PYG@four\else
+    \ifx\UTFviii@three@octets#1\let\next=\spx@verb@wrapPYG@PYG@three\else
+    \ifx\UTFviii@two@octets#1\let\next=\spx@verb@wrapPYG@PYG@two\else
+    \let\next=\spx@verb@wrapPYG@PYG@one
+    \fi\fi\fi
+    \next
+}%
+\long\def\spx@verb@wrapPYG@PYG@one#1{%
+    \expandafter\PYG\spx@verb@wrapPYG@PYG@spec{#1}%
+    \futurelet\spx@nexttoken\spx@verb@wrapPYG@PYG@i
+}%
+\long\def\spx@verb@wrapPYG@PYG@two#1#2{%
+    \expandafter\PYG\spx@verb@wrapPYG@PYG@spec{#1#2}%
+    \futurelet\spx@nexttoken\spx@verb@wrapPYG@PYG@i
+}%
+\long\def\spx@verb@wrapPYG@PYG@three#1#2#3{%
+    \expandafter\PYG\spx@verb@wrapPYG@PYG@spec{#1#2#3}%
+    \futurelet\spx@nexttoken\spx@verb@wrapPYG@PYG@i
+}%
+\long\def\spx@verb@wrapPYG@PYG@four#1#2#3#4{%
+    \expandafter\PYG\spx@verb@wrapPYG@PYG@spec{#1#2#3#4}%
+    \futurelet\spx@nexttoken\spx@verb@wrapPYG@PYG@i
+}%
+\catcode`Z 11%
+%
 \g@addto@macro\FV@SetupFont{%
     \sbox\sphinxcontinuationbox {\spx@opt@verbatimcontinued}%
     \sbox\sphinxvisiblespacebox {\spx@opt@verbatimvisiblespace}%