docs: add :downloadUrls: role

Also added support for <downloadurl> in lexers (sh, shell-session).

11 files changed, 229 insertions, 118 deletions

diff --git a/docs/manuals/source/DocumentationStyleGuide/BareosSpecificFormatting.rst b/docs/manuals/source/DocumentationStyleGuide/BareosSpecificFormatting.rst
index 6ccee88a1..4833eb875 100644
--- a/docs/manuals/source/DocumentationStyleGuide/BareosSpecificFormatting.rst
+++ b/docs/manuals/source/DocumentationStyleGuide/BareosSpecificFormatting.rst
@@ -9,5 +9,5 @@ Bareos Specific Formatting
    /DocumentationStyleGuide/BareosSpecificFormatting/Bconsole.rst
    /DocumentationStyleGuide/BareosSpecificFormatting/BareosLogging.rst
    /DocumentationStyleGuide/BareosSpecificFormatting/Limitation.rst
-   /DocumentationStyleGuide/BareosSpecificFormatting/Issue.rst
+   /DocumentationStyleGuide/BareosSpecificFormatting/Links.rst
    /DocumentationStyleGuide/BareosSpecificFormatting/Release.rst
diff --git a/docs/manuals/source/DocumentationStyleGuide/BareosSpecificFormatting/Issue.rst b/docs/manuals/source/DocumentationStyleGuide/BareosSpecificFormatting/Issue.rst
deleted file mode 100644
index 9f4d78b5f..000000000
--- a/docs/manuals/source/DocumentationStyleGuide/BareosSpecificFormatting/Issue.rst
+++ /dev/null
@@ -1,42 +0,0 @@
-Issues tracked on Mantis
-========================
-
-bugs.bareos.org
----------------
-
-If you want to reference an issue that is tracked on https://bugs.bareos.org/, the following formatting should be used:
-
-.. literalinclude:: /DocumentationStyleGuide/example/mantis-issue.rst.inc
-   :language: none
-
-This will be displayed as:
-
-.. include:: /DocumentationStyleGuide/example/mantis-issue.rst.inc
-
-Github
-------
-
-Pull request
-~~~~~~~~~~~~
-
-Refer to a Github Pull request by:
-
-.. literalinclude:: /DocumentationStyleGuide/example/github-pull-request.rst.inc
-   :language: none
-
-This will be displayed as:
-
-.. include:: /DocumentationStyleGuide/example/github-pull-request.rst.inc
-
-
-Commit
-~~~~~~
-
-Refer to a Github Commit by:
-
-.. literalinclude:: /DocumentationStyleGuide/example/github-commit.rst.inc
-   :language: none
-
-This will be displayed as:
-
-.. include:: /DocumentationStyleGuide/example/github-commit.rst.inc
diff --git a/docs/manuals/source/DocumentationStyleGuide/BareosSpecificFormatting/Links.rst b/docs/manuals/source/DocumentationStyleGuide/BareosSpecificFormatting/Links.rst
new file mode 100644
index 000000000..f44fc73e9
--- /dev/null
+++ b/docs/manuals/source/DocumentationStyleGuide/BareosSpecificFormatting/Links.rst
@@ -0,0 +1,73 @@
+.. _documentationstyleguide/bareosspecificformatting/links:
+
+Links
+=====
+
+bugs.bareos.org
+---------------
+
+If you want to reference an issue that is tracked on https://bugs.bareos.org/, the following formatting should be used:
+
+.. literalinclude:: /DocumentationStyleGuide/example/mantis-issue.rst.inc
+   :language: none
+
+This will be displayed as:
+
+.. include:: /DocumentationStyleGuide/example/mantis-issue.rst.inc
+
+Github
+------
+
+Pull request
+~~~~~~~~~~~~
+
+Refer to a Github Pull request by:
+
+.. literalinclude:: /DocumentationStyleGuide/example/github-pull-request.rst.inc
+   :language: none
+
+This will be displayed as:
+
+.. include:: /DocumentationStyleGuide/example/github-pull-request.rst.inc
+
+
+Commit
+~~~~~~
+
+Refer to a Github Commit by:
+
+.. literalinclude:: /DocumentationStyleGuide/example/github-commit.rst.inc
+   :language: none
+
+This will be displayed as:
+
+.. include:: /DocumentationStyleGuide/example/github-commit.rst.inc
+
+
+Download Urls
+-------------
+
+The role ``:downloadUrls:`` expands to the download URL of the current build. It uses variables set in Jenkins for this.
+Some downloads are provided from https://download.bareos.org/, others from https://download.bareos.com/. The role detects if a package will be published on download.bareos.org only or additionally on download.bareos.com.
+
+If a release is on both download servers, it will combine them with **or**.
+
+.. literalinclude:: /DocumentationStyleGuide/example/downloadUrls.rst.inc
+   :language: none
+   
+will be displayed as:
+
+https://download.bareos.org/bareos/release/21/Debian_11/ or https://download.bareos.com/bareos/release/21/Debian_11/
+
+For the current build, the output is:
+
+.. include:: /DocumentationStyleGuide/example/downloadUrls.rst.inc
+
+If only the base release URL should be displayed, use **.** as parameter:
+
+.. literalinclude:: /DocumentationStyleGuide/example/downloadUrls-none.rst.inc
+   :language: none
+
+The output will look like this:
+
+.. include:: /DocumentationStyleGuide/example/downloadUrls-none.rst.inc
diff --git a/docs/manuals/source/DocumentationStyleGuide/BareosSpecificFormatting/Release.rst b/docs/manuals/source/DocumentationStyleGuide/BareosSpecificFormatting/Release.rst
index 91694c3e4..0ad30e61b 100644
--- a/docs/manuals/source/DocumentationStyleGuide/BareosSpecificFormatting/Release.rst
+++ b/docs/manuals/source/DocumentationStyleGuide/BareosSpecificFormatting/Release.rst
@@ -53,37 +53,3 @@ Note
   * https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html#directive-deprecated
 
 Until defined otherwise, you should use ``:sinceVersion:`` instead of these.
-
-
-URLs
-----
-
-Release URL - Bareos.org
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-If you want to display the release download URL from the Bareos.org site, the following formatting should be used:
-
-.. \newcommand{\releaseUrlDownloadBareosOrg}[1]{%
-   \url{http://download.bareos.org/bareos/release/#1/}%
-   }
-
-   Post Conversion Changes
-   ${PERL} 's#\\releaseUrlDownloadBareosOrg\{(.*?)\}#//http://download.bareos.org/bareos/release/\1/#g' ${DESTFILE}
-
-http://download.bareos.org/bareos/release/18.2/
-
-
-Release URL - Bareos.com
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-If you want to display the release download URL from the Bareos.com site, the following formatting should be used:
-
-.. \newcommand{\releaseUrlDownloadBareosOrg}[1]{%
-   \url{http://download.bareos.org/bareos/release/#1/}%
-   }
-
-   Post Conversion Changes
-   ${PERL} 's#\\releaseUrlDownloadBareosCom\{(.*?)\}#//https://download.bareos.com/bareos/release/\1/#g' ${DESTFILE}
-
-
-https://download.bareos.com/bareos/release/18.2/
diff --git a/docs/manuals/source/DocumentationStyleGuide/RestOverview.rst b/docs/manuals/source/DocumentationStyleGuide/RestOverview.rst
index 353b9a3b4..bc79a3120 100644
--- a/docs/manuals/source/DocumentationStyleGuide/RestOverview.rst
+++ b/docs/manuals/source/DocumentationStyleGuide/RestOverview.rst
@@ -261,6 +261,7 @@ Output:
 
 .. include:: /DocumentationStyleGuide/example/urls.rst.inc
 
+There are also some roles for specific links targets, see :ref:`documentationstyleguide/bareosspecificformatting/links`.
 
 Internal links
 ~~~~~~~~~~~~~~
diff --git a/docs/manuals/source/DocumentationStyleGuide/example/downloadUrls-none.rst.inc b/docs/manuals/source/DocumentationStyleGuide/example/downloadUrls-none.rst.inc
new file mode 100644
index 000000000..9b35131e0
--- /dev/null
+++ b/docs/manuals/source/DocumentationStyleGuide/example/downloadUrls-none.rst.inc
@@ -0,0 +1 @@
+:downloadUrls:`.`
diff --git a/docs/manuals/source/DocumentationStyleGuide/example/downloadUrls.rst.inc b/docs/manuals/source/DocumentationStyleGuide/example/downloadUrls.rst.inc
new file mode 100644
index 000000000..1dd7731a2
--- /dev/null
+++ b/docs/manuals/source/DocumentationStyleGuide/example/downloadUrls.rst.inc
@@ -0,0 +1 @@
+:downloadUrls:`Debian_11/`
diff --git a/docs/manuals/source/_extensions/bareos-ext.py b/docs/manuals/source/_extensions/bareos-ext.py
index ee7bbbee9..39ba1ad1a 100644
--- a/docs/manuals/source/_extensions/bareos-ext.py
+++ b/docs/manuals/source/_extensions/bareos-ext.py
@@ -31,6 +31,7 @@ from sphinx.util.docfields import Field
 from docutils import nodes
 from pprint import pformat
 import re
+from bareos_urls import BareosUrls
 
 # import logging
 
@@ -694,6 +695,37 @@ def sinceVersion():
     return role
 
 
+def downloadUrls():
+    def role(name, rawtext, text, lineno, inliner, options={}, content=[]):
+
+        urls = BareosUrls()
+
+        distribution = ""
+        if text.lower() != "none" and text != ".":
+            distribution = text
+
+        download_bareos_org_url = urls.get_download_bareos_org_url(distribution)
+        download_bareos_com_url = urls.get_download_bareos_com_url(distribution)
+
+        org_url = nodes.reference(
+            rawtext, download_bareos_org_url, refuri=download_bareos_org_url, **options
+        )
+
+        if download_bareos_com_url is not None:
+            text_or = nodes.Text(" or ")
+            com_url = nodes.reference(
+                rawtext,
+                download_bareos_com_url,
+                refuri=download_bareos_com_url,
+                **options
+            )
+            return [org_url, text_or, com_url], []
+        else:
+            return [org_url], []
+
+    return role
+
+
 def setup(app):
     # logging.basicConfig(filename="/tmp/build/sphinx-bareos.log", level=logging.DEBUG)
     # logger = logging.getLogger(__name__)
@@ -703,13 +735,10 @@ def setup(app):
     app.add_role("bcommand", bcommand())
     app.add_role("os", os())
     app.add_role("sinceversion", sinceVersion())
+    app.add_role("downloadurls", downloadUrls())
     app.add_role(
         "mantis", autolink("https://bugs.bareos.org/view.php?id={}", "Issue #{}")
     )
 
     # identifies the version of our extension
-    return {
-        "version": "0.4",
-        "parallel_read_safe": False,
-        "parallel_write_safe": True
-    }
+    return {"version": "0.4", "parallel_read_safe": False, "parallel_write_safe": True}
diff --git a/docs/manuals/source/_extensions/bareos_lexers.py b/docs/manuals/source/_extensions/bareos_lexers.py
index bedae50bd..91e27b6d6 100644
--- a/docs/manuals/source/_extensions/bareos_lexers.py
+++ b/docs/manuals/source/_extensions/bareos_lexers.py
@@ -1,6 +1,6 @@
 #   BAREOS - Backup Archiving REcovery Open Sourced
 #
-#   Copyright (C) 2019-2020 Bareos GmbH & Co. KG
+#   Copyright (C) 2019-2022 Bareos GmbH & Co. KG
 #
 #   This program is Free Software; you can redistribute it and/or
 #   modify it under the terms of version three of the GNU Affero General Public
@@ -18,8 +18,9 @@
 #   02110-1301, USA.
 
 from pygments.lexer import RegexLexer, inherit, bygroups
-from pygments.lexers.shell import BashLexer
+from pygments.lexers.shell import BashLexer, BashSessionLexer
 from pygments.token import *
+from bareos_urls import BareosUrls
 
 #
 # http://pygments.org/docs/lexerdevelopment/
@@ -27,24 +28,35 @@ from pygments.token import *
 #
 
 # Test:
-#  cat consolidate.cfg | pygmentize -l extensions/bareos_lexers.py:BareosConfigLexer -x
+#  cat consolidate.cfg | pygmentize -l _extensions/bareos_lexers.py:BareosConfigLexer -x
 #
 
 
 class BareosBaseLexer(BashLexer):
     name = "BareosBase"
 
+    def downloadurl_callback(lexer, match):
+        yield match.start(), Generic.Headline, BareosUrls().get_download_bareos_org_url()
+
     tokens = {
         "root": [
             # (r'(<input>)(.*)(</input>)', bygroups(None, Generic.Emph, None)),
             # (r'(<input>)(.*)(</input>)', bygroups(None, Generic.Strong, None)),
             (r"(<input>)(.*)(</input>)", bygroups(None, Generic.Heading, None)),
             (r"(<strong>)(.*)(</strong>)", bygroups(None, Generic.Strong, None)),
+            (r"(<downloadurl>)", downloadurl_callback),
             inherit,
         ]
     }
 
 
+class BareosShellLexer(BareosBaseLexer):
+    name = "ShellBareos"
+    aliases = ["sh"]
+
+    tokens = {"root": [inherit]}
+
+
 class BareosConfigLexer(BareosBaseLexer):
     name = "BareosConfig"
     aliases = ["bareosconfig", "bconfig"]
@@ -75,3 +87,13 @@ class BareosMessageLexer(BareosBaseLexer):
     # filenames = ['*.log']
 
     tokens = {"root": [inherit]}
+
+
+class BareosShellSessionLexer(BashSessionLexer):
+    name = "BareosShellSession"
+    aliases = ["shell-session"]
+
+    def get_tokens_unprocessed(self, text):
+        url = BareosUrls().get_download_bareos_org_url()
+        text = text.replace("<downloadurl>", url)
+        return super(BareosShellSessionLexer, self).get_tokens_unprocessed(text)
diff --git a/docs/manuals/source/_extensions/bareos_urls.py b/docs/manuals/source/_extensions/bareos_urls.py
new file mode 100644
index 000000000..2332636b0
--- /dev/null
+++ b/docs/manuals/source/_extensions/bareos_urls.py
@@ -0,0 +1,58 @@
+#   BAREOS - Backup Archiving REcovery Open Sourced
+#
+#   Copyright (C) 2022-2022 Bareos GmbH & Co. KG
+#
+#   This program is Free Software; you can redistribute it and/or
+#   modify it under the terms of version three of the GNU Affero General Public
+#   License as published by the Free Software Foundation and included
+#   in the file LICENSE.
+#
+#   This program is distributed in the hope that it will be useful, but
+#   WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+#   Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program; if not, write to the Free Software
+#   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+#   02110-1301, USA.
+
+import os
+
+
+class BareosUrls(object):
+    def __init__(self):
+        release_variant = os.environ.get("RELEASE_VARIANT", "prerelease")
+        branch_name = os.environ.get("BRANCH_NAME", "")
+        publish_repo_path = os.environ.get("PUBLISH_REPO_PATH", "experimental/nightly")
+
+        self.download_bareos_org_url = "https://download.bareos.org/bareos/{}/".format(
+            publish_repo_path
+        )
+        self.download_bareos_com_url = "https://download.bareos.com/bareos/{}/".format(
+            publish_repo_path
+        )
+
+        # if release_variant == 'subscription':
+        # # org + com
+        # elif release_variant == 'community':
+        # # org
+        # else:
+        # # env.RELEASE_VARIANT == 'prerelease'
+        # if branch_name.startswith("bareos-"):
+        # # org + com
+        # else:
+        # # org
+
+        self.download_bareos_com = False
+        if release_variant == "subscription" or branch_name.startswith("bareos-"):
+            self.download_bareos_com = True
+
+    def get_download_bareos_org_url(self, tail=""):
+        return self.download_bareos_org_url + tail
+
+    def get_download_bareos_com_url(self, tail="", force=False):
+        if force or self.download_bareos_com:
+            return self.download_bareos_com_url + tail
+        else:
+            return None
diff --git a/docs/manuals/source/conf.py b/docs/manuals/source/conf.py
index 99c4c24a2..e1542a086 100644
--- a/docs/manuals/source/conf.py
+++ b/docs/manuals/source/conf.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 #   BAREOS - Backup Archiving REcovery Open Sourced
 #
-#   Copyright (C) 2018-2021 Bareos GmbH & Co. KG
+#   Copyright (C) 2018-2022 Bareos GmbH & Co. KG
 #
 #   This program is Free Software; you can redistribute it and/or
 #   modify it under the terms of version three of the GNU Affero General Public
@@ -302,52 +302,52 @@ latex_documents = [
 
 linkcheck_allowed_redirects = {
     # All HTTP redirections from the source URI to the canonical URI will be treated as "working".
-    r'https://bareos\.org/.*': r'https://bareos\.com/.*',
-    r'http://www\.bareos\.org': r'https://www\.bareos\.com/.*',
-    r'http://www\.bareos\.org/.*': r'https://www\.bareos\.com/.*',
-    r'https://www\.bareos\.org/.*': r'https://www\.bareos\.com/.*',
-    r'https://www\.bareos\.com/whitepapers/.*': r'https://www\.bareos\.com/.*',
-    r'https://bugs\.bareos\.org/.*': r'https://bugs\.bareos\.org/my_view_page\.php',
-    r'https://download\.bareos\.org': r'https://download\.bareos\.org/bareos',
-    r'https://download\.bareos\.com': r'https://download\.bareos\.com/bareos',
-    r'https://github.com/markh794/mhvtl.git': r'https://github.com/markh794/mhvtl',
+    r"https://bareos\.org/.*": r"https://bareos\.com/.*",
+    r"http://www\.bareos\.org": r"https://www\.bareos\.com/.*",
+    r"http://www\.bareos\.org/.*": r"https://www\.bareos\.com/.*",
+    r"https://www\.bareos\.org/.*": r"https://www\.bareos\.com/.*",
+    r"https://www\.bareos\.com/whitepapers/.*": r"https://www\.bareos\.com/.*",
+    r"https://bugs\.bareos\.org/.*": r"https://bugs\.bareos\.org/my_view_page\.php",
+    r"https://download\.bareos\.org": r"https://download\.bareos\.org/bareos",
+    r"https://download\.bareos\.com": r"https://download\.bareos\.com/bareos",
+    r"https://github.com/markh794/mhvtl.git": r"https://github.com/markh794/mhvtl",
 }
 linkcheck_request_headers = {
-#    "https://www.sphinx-doc.org/": {
-#        "Accept": "text/html",
-#        "Accept-Encoding": "utf-8",
-#    },
+    #    "https://www.sphinx-doc.org/": {
+    #        "Accept": "text/html",
+    #        "Accept-Encoding": "utf-8",
+    #    },
     "*": {
         "Accept": "text/html,application/xhtml+xml",
     }
 }
-linkcheck_retries = 2   # default 1
+linkcheck_retries = 2  # default 1
 linkcheck_timeout = 10  # default 300
-linkcheck_workers = 2   # default 5
+linkcheck_workers = 2  # default 5
 linkcheck_anchors = True
 linkcheck_ignore = [
-    r'\.\./*',
-    r'http://localhost:*/',
-    r'https://\w+:\d+/bareos-webui/',
-    'https://UCS_SERVER/bareos-webui/',
-    'http://HOSTNAME/bareos-webui',
-    'http://localhost:9100',
-    'http://bareos:9100',
-    'http://bucket.s3_server/object',
-    'http://127.0.0.1:8000/docs',
-    'http://127.0.0.1:8000/redoc',
-    'https://pubs.vmware.com/vsphere-55/topic/com.vmware.vsphere.security.doc/*',
-    r'https://github\.com/bareos/bareos/pull/\d+', 
-    'https://www.glusterfs.org',                     #failing too often
-    'https://www.sphinx-doc.org/en/1.7/intro.html#',
+    r"\.\./*",
+    r"http://localhost:*/",
+    r"https://\w+:\d+/bareos-webui/",
+    "https://UCS_SERVER/bareos-webui/",
+    "http://HOSTNAME/bareos-webui",
+    "http://localhost:9100",
+    "http://bareos:9100",
+    "http://bucket.s3_server/object",
+    "http://127.0.0.1:8000/docs",
+    "http://127.0.0.1:8000/redoc",
+    "https://pubs.vmware.com/vsphere-55/topic/com.vmware.vsphere.security.doc/*",
+    r"https://github\.com/bareos/bareos/pull/\d+",
+    "https://www.glusterfs.org",  # failing too often
+    "https://www.sphinx-doc.org/en/1.7/intro.html#",
 ]
-#linkcheck_auth = [
+# linkcheck_auth = [
 #  ('https://foo\.yourcompany\.com/.+', ('johndoe', 'secret')),
 #  ('https://.+\.yourcompany\.com/.+', HTTPDigestAuth(...)),
-#]
+# ]
 linkcheck_rate_limit_timeout = 300.00
 # ignore all links in documents located in a subfolder named 'legacy'
-#linkcheck_exclude_documents = [r'.*/legacy/.*']
+# linkcheck_exclude_documents = [r'.*/legacy/.*']
 
 # -- Options for manual page output ---------------------------------------
 
@@ -412,6 +412,8 @@ lexers["bareosconfig"] = BareosConfigLexer()
 lexers["bconsole"] = BareosConsoleLexer()
 lexers["bareoslog"] = BareosLogLexer()
 lexers["bareosmessage"] = BareosMessageLexer()
+lexers["sh"] = BareosShellLexer()
+lexers["shell-session"] = BareosShellSessionLexer()
 
 
 # generate rst.inc files from json files