Various optimizations part 1

* Configure for python3
* Update tests
* Use appropriate virtualenv
* Install nginx for the integration tests
* Try use LD_LIBRARY_PATH to find augeas shared library in snap when python-augeas is invoked
* Update travis to use build-in setup capabilities
* Update .travis.yml
* Add acme build
* Update tests
* Try more recent dist
* Update command
* Clean tests
* Add back augeas
* Add env
* Revert to last working snapcraft config
* Add a gitignore
* Reintegrate acme. Declare augeas in certbot parts
* Use release version of certbot
* Try new approach
* Fix config
* Directly install version of python-augeas from pypi
* Restart from basic
* Clone only once certbot repository. Use pinned versions of dependencies from certbot-auto.
* Try relatively to source
* Use snapcraft env variables
* Strip hashes
* Fix path
* Redefine path
* Continue to prepare the runtime
* Fix command line
* Update .travis.yml
* Add back certbot-apache
* Update snapcraft.yaml
* Build snap against the latest release of certbot

6 files changed, 80 insertions, 59 deletions

diff --git a/snap/.gitignore b/snap/.gitignore
new file mode 100644
index 000000000..2a339aeaa
--- /dev/null
+++ b/snap/.gitignore
@@ -0,0 +1,8 @@
+.snapcraft
+parts
+prime
+stage
+*.snap
+certbot
+venv
+.idea
diff --git a/snap/.travis.yml b/snap/.travis.yml
index 603609b6d..1abe677cd 100644
--- a/snap/.travis.yml
+++ b/snap/.travis.yml
@@ -1,11 +1,29 @@
+dist: xenial
 language: bash
-script: docker run -v $(pwd):$(pwd) -t snapcore/snapcraft sh -c "apt-get update -qq && cd $(pwd) && snapcraft" && sudo apt-get -y install snapd && sudo snap install --dangerous --classic *.snap && sudo bash test.bash
+install:
+  - EXPORT CERTBOT_VERSION=$(python3 get_latest_version.py)
+  - git clone https://github.com/certbot/certbot.git certbot --branch ${CERTBOT_VERSION}
+  - certbot/tools/strip_hashes.py certbot/letsencrypt-auto-source/pieces/dependency-requirements.txt > certbot/constraints.txt
+script:
+  - docker run -v $(pwd):$(pwd) -t -e CERTBOT_VERSION=${CERTBOT_VERSION} snapcore/snapcraft sh -c "apt-get update -qq && apt-get install -qq git && cd $(pwd) && snapcraft"
+  - sudo snap install --dangerous --classic *.snap
+  - sudo bash test.sh
 sudo: required
+addons:
+  apt:
+    packages:
+    - snapd
+    - nginx-light
+    - python3-venv
+    - python3-requests
+    - python3-packaging
 services:
-- docker
+  - docker
 deploy:
   'on':
     branch: master
   provider: script
-  script: openssl aes-256-cbc -K $encrypted_edc9d3b1405a_key -iv $encrypted_edc9d3b1405a_iv -in .snapcraft/travis_snapcraft.cfg -out .snapcraft/snapcraft.cfg -d && docker run -v $(pwd):$(pwd) -t snapcore/snapcraft sh -c "cd $(pwd) && snapcraft push *.snap --release edge"
+  script: |
+    openssl aes-256-cbc -K $encrypted_edc9d3b1405a_key -iv $encrypted_edc9d3b1405a_iv -in .snapcraft/travis_snapcraft.cfg -out .snapcraft/snapcraft.cfg -d
+    docker run -v $(pwd):$(pwd) -t snapcore/snapcraft sh -c "cd $(pwd) && snapcraft push *.snap --release edge"
   skip_cleanup: true
diff --git a/snap/get_latest_version.py b/snap/get_latest_version.py
new file mode 100644
index 000000000..b9a57e2aa
--- /dev/null
+++ b/snap/get_latest_version.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+import sys
+
+import requests
+from packaging import version
+
+
+def main():
+    result = requests.get('https://api.github.com/repos/certbot/certbot/tags')
+    result.raise_for_status()
+
+    tags = [version.parse(entry['name'].replace('v', '')) for entry in result.json()]
+    tags.sort()
+
+    latest_tag = 'v{0}'.format(tags[-1])
+
+    sys.stdout.write(latest_tag)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml
index 46425f182..0a3772ad6 100644
--- a/snap/snapcraft.yaml
+++ b/snap/snapcraft.yaml
@@ -2,6 +2,7 @@ name: certbot
 version: script
 version-script: cd parts/certbot/src && git describe|sed s/^v//
 summary: Automatically configure HTTPS using Let's Encrypt
+base: core18
 description: |
  The objective of Certbot, Let's Encrypt, and the ACME (Automated
  Certificate Management Environment) protocol is to make it possible
@@ -23,45 +24,42 @@ apps:
     command: certbot
     environment:
       PATH: "$SNAP/bin:$SNAP/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games"
-      AUGEAS_LENS_LIB: $SNAP/usr/share/augeas/lenses/dist
+      AUGEAS_LENS_LIB: "$SNAP/usr/share/augeas/lenses/dist"
+      LD_LIBRARY_PATH: "$SNAP/usr/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH"
   renew:
     command: certbot -q renew
     daemon: oneshot
     environment:
       PATH: "$SNAP/bin:$SNAP/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games"
       AUGEAS_LENS_LIB: $SNAP/usr/share/augeas/lenses/dist
+      LD_LIBRARY_PATH: "$SNAP/usr/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH"
     passthrough:
         # Run approximately twice a day with randomization
         timer: 00:00~24:00/2
 
 parts:
-  python-augeas:
+  acme:
     plugin: python
-    source: git://github.com/basak/python-augeas
-    source-branch: snap
-    python-version: python2
-    build-packages: [libaugeas0]
+    source: certbot
+    source-subdir: acme
+    constraints: $SNAPCRAFT_PART_SRC/constraints.txt
+    python-version: python3
   certbot:
     plugin: python
-    source: git://github.com/certbot/certbot
-    source-branch: master
-    python-version: python2
-    after: [python-augeas]
-  certbot-nginx:
-    plugin: python
-    source: git://github.com/certbot/certbot
-    source-branch: master
-    source-subdir: certbot-nginx
-    build-packages: [libaugeas-dev]
-    stage-packages: [libaugeas0]
-    after: [certbot]
-    python-version: python2
+    source: certbot
+    constraints: $SNAPCRAFT_PART_SRC/constraints.txt
+    python-version: python3
+    python-packages:
+    - acme
+    after: [acme]
   certbot-apache:
     plugin: python
-    source: git://github.com/certbot/certbot
-    source-branch: master
+    source: certbot
     source-subdir: certbot-apache
+    constraints: $SNAPCRAFT_PART_SRC/constraints.txt
+    python-version: python3
+    python-packages:
+    - certbot
     build-packages: [libaugeas-dev]
     stage-packages: [libaugeas0]
     after: [certbot]
-    python-version: python2
diff --git a/snap/test.bash b/snap/test.bash
deleted file mode 100644
index e723b67bf..000000000
--- a/snap/test.bash
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-apt-get -y install lsb-release net-tools wget python nginx
-
-wget https://github.com/docker/compose/releases/download/1.15.0-rc1/docker-compose-Linux-x86_64 -O /usr/local/bin/docker-compose
-chmod +x /usr/local/bin/docker-compose
-
-cat << EOF > /usr/local/bin/coverage
-#!/bin/bash -xe
-
-if [ "\$1" != "run" ]; then
-    exit 0;
-fi
-
-"\${@:7}"
-EOF
-chmod +x /usr/local/bin/coverage
-
-certbot_version=$(certbot --version 2>&1 | grep "^certbot" | cut -d " " -f 2)
-
-cd parts/certbot/src
-
-tests/boulder-fetch.sh
-until curl http://localhost:4000/directory 2>/dev/null; do
-  echo waiting for boulder
-  sleep 1
-done
-# Not needed under Travis Trusty?
-#sed -i "s/'1.3.6.1.5.5.7.1.24'/-e '1.3.6.1.5.5.7.1.24' -e 'status_request'/g" tests/certbot-boulder-integration.sh
-tests/boulder-integration.sh
-
-echo "Success!"
diff --git a/snap/test.sh b/snap/test.sh
new file mode 100644
index 000000000..cf2418448
--- /dev/null
+++ b/snap/test.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+set -ex
+
+cd certbot
+
+python3 -m venv venv
+. venv/bin/activate
+pip install -e certbot-ci
+
+pytest certbot-ci/certbot_integration_tests/certbot_tests --numprocesses 4 --acme-server=pebble