From d19698251da19f363d94ece9e5ee9dcc425afad2 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Tue, 31 Jul 2018 17:08:39 +0300 Subject: Do not send status or resource fields in newOrder payloads for ACMEv2 --- acme/acme/client.py | 4 ++-- acme/acme/messages.py | 23 +++++++++++++++-------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index bd86657b9..60a67a038 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -646,7 +646,7 @@ class ClientV2(ClientBase): value=name)) order = messages.NewOrder(identifiers=identifiers) response = self._post(self.directory['newOrder'], order) - body = messages.Order.from_json(response.json()) + body = messages.OrderBase.from_json(response.json()) authorizations = [] for url in body.authorizations: authorizations.append(self._authzr_from_response(self.net.get(url), uri=url)) @@ -715,7 +715,7 @@ class ClientV2(ClientBase): while datetime.datetime.now() < deadline: time.sleep(1) response = self.net.get(orderr.uri) - body = messages.Order.from_json(response.json()) + body = messages.OrderBase.from_json(response.json()) if body.error is not None: raise errors.IssuanceError(body.error) if body.certificate is not None: diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 5be458580..86585ccb9 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -509,11 +509,10 @@ class Revocation(jose.JSONObjectWithFields): reason = jose.Field('reason') -class Order(ResourceBody): +class OrderBase(ResourceBody): """Order Resource Body. :ivar list of .Identifier: List of identifiers for the certificate. - :ivar acme.messages.Status status: :ivar list of str authorizations: URLs of authorizations. :ivar str certificate: URL to download certificate as a fullchain PEM. :ivar str finalize: URL to POST to to request issuance once all @@ -522,8 +521,6 @@ class Order(ResourceBody): :ivar .Error error: Any error that occurred during finalization, if applicable. """ identifiers = jose.Field('identifiers', omitempty=True) - status = jose.Field('status', decoder=Status.from_json, - omitempty=True, default=STATUS_PENDING) authorizations = jose.Field('authorizations', omitempty=True) certificate = jose.Field('certificate', omitempty=True) finalize = jose.Field('finalize', omitempty=True) @@ -534,6 +531,16 @@ class Order(ResourceBody): def identifiers(value): # pylint: disable=missing-docstring,no-self-argument return tuple(Identifier.from_json(identifier) for identifier in value) + +class Order(OrderBase): + """Order Resource Body for ACMEv1 + + :ivar acme.messages.Status status: + """ + status = jose.Field('status', decoder=Status.from_json, + omitempty=True, default=STATUS_PENDING) + + class OrderResource(ResourceWithURI): """Order Resource. @@ -549,8 +556,8 @@ class OrderResource(ResourceWithURI): authorizations = jose.Field('authorizations') fullchain_pem = jose.Field('fullchain_pem', omitempty=True) + @Directory.register -class NewOrder(Order): - """New order.""" - resource_type = 'new-order' - resource = fields.Resource(resource_type) +class NewOrder(OrderBase): + """New order for ACMEv2""" + resource_type = "new-order" -- cgit v1.2.3 From 8b3629ebd49b647967dde9e279ad252583f6b881 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Tue, 31 Jul 2018 19:55:19 +0300 Subject: Fix tests --- acme/acme/client_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 4f8a1abe2..3cdcdf041 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -695,9 +695,8 @@ class ClientV2Test(ClientTestBase): self.authzr2 = messages.AuthorizationResource( body=self.authz2, uri=self.authzr_uri2) - self.order = messages.Order( + self.order = messages.OrderBase( identifiers=(self.authz.identifier, self.authz2.identifier), - status=messages.STATUS_PENDING, authorizations=(self.authzr.uri, self.authzr_uri2), finalize='https://www.letsencrypt-demo.org/acme/acct/1/order/1/finalize') self.orderr = messages.OrderResource( -- cgit v1.2.3 From c131f4211de987cdd7738a90c091fb25dce54351 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 1 Aug 2018 12:10:01 +0300 Subject: Revert "Fix tests" This reverts commit 8b3629ebd49b647967dde9e279ad252583f6b881. --- acme/acme/client_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 3cdcdf041..4f8a1abe2 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -695,8 +695,9 @@ class ClientV2Test(ClientTestBase): self.authzr2 = messages.AuthorizationResource( body=self.authz2, uri=self.authzr_uri2) - self.order = messages.OrderBase( + self.order = messages.Order( identifiers=(self.authz.identifier, self.authz2.identifier), + status=messages.STATUS_PENDING, authorizations=(self.authzr.uri, self.authzr_uri2), finalize='https://www.letsencrypt-demo.org/acme/acct/1/order/1/finalize') self.orderr = messages.OrderResource( -- cgit v1.2.3 From b1b46508045d90715e3043ad9e373ba43edf9a8f Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 1 Aug 2018 12:10:55 +0300 Subject: Revert "Do not send status or resource fields in newOrder payloads for ACMEv2" This reverts commit d19698251da19f363d94ece9e5ee9dcc425afad2. --- acme/acme/client.py | 4 ++-- acme/acme/messages.py | 23 ++++++++--------------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 60a67a038..bd86657b9 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -646,7 +646,7 @@ class ClientV2(ClientBase): value=name)) order = messages.NewOrder(identifiers=identifiers) response = self._post(self.directory['newOrder'], order) - body = messages.OrderBase.from_json(response.json()) + body = messages.Order.from_json(response.json()) authorizations = [] for url in body.authorizations: authorizations.append(self._authzr_from_response(self.net.get(url), uri=url)) @@ -715,7 +715,7 @@ class ClientV2(ClientBase): while datetime.datetime.now() < deadline: time.sleep(1) response = self.net.get(orderr.uri) - body = messages.OrderBase.from_json(response.json()) + body = messages.Order.from_json(response.json()) if body.error is not None: raise errors.IssuanceError(body.error) if body.certificate is not None: diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 86585ccb9..5be458580 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -509,10 +509,11 @@ class Revocation(jose.JSONObjectWithFields): reason = jose.Field('reason') -class OrderBase(ResourceBody): +class Order(ResourceBody): """Order Resource Body. :ivar list of .Identifier: List of identifiers for the certificate. + :ivar acme.messages.Status status: :ivar list of str authorizations: URLs of authorizations. :ivar str certificate: URL to download certificate as a fullchain PEM. :ivar str finalize: URL to POST to to request issuance once all @@ -521,6 +522,8 @@ class OrderBase(ResourceBody): :ivar .Error error: Any error that occurred during finalization, if applicable. """ identifiers = jose.Field('identifiers', omitempty=True) + status = jose.Field('status', decoder=Status.from_json, + omitempty=True, default=STATUS_PENDING) authorizations = jose.Field('authorizations', omitempty=True) certificate = jose.Field('certificate', omitempty=True) finalize = jose.Field('finalize', omitempty=True) @@ -531,16 +534,6 @@ class OrderBase(ResourceBody): def identifiers(value): # pylint: disable=missing-docstring,no-self-argument return tuple(Identifier.from_json(identifier) for identifier in value) - -class Order(OrderBase): - """Order Resource Body for ACMEv1 - - :ivar acme.messages.Status status: - """ - status = jose.Field('status', decoder=Status.from_json, - omitempty=True, default=STATUS_PENDING) - - class OrderResource(ResourceWithURI): """Order Resource. @@ -556,8 +549,8 @@ class OrderResource(ResourceWithURI): authorizations = jose.Field('authorizations') fullchain_pem = jose.Field('fullchain_pem', omitempty=True) - @Directory.register -class NewOrder(OrderBase): - """New order for ACMEv2""" - resource_type = "new-order" +class NewOrder(Order): + """New order.""" + resource_type = 'new-order' + resource = fields.Resource(resource_type) -- cgit v1.2.3 From 8943dffe0d804eba8da99a77c5b8b4b72cce8991 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 1 Aug 2018 12:17:23 +0300 Subject: Removed status and resource fields from NewOrder object --- acme/acme/client_test.py | 1 - acme/acme/messages.py | 4 ---- 2 files changed, 5 deletions(-) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 4f8a1abe2..965ece55d 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -697,7 +697,6 @@ class ClientV2Test(ClientTestBase): self.order = messages.Order( identifiers=(self.authz.identifier, self.authz2.identifier), - status=messages.STATUS_PENDING, authorizations=(self.authzr.uri, self.authzr_uri2), finalize='https://www.letsencrypt-demo.org/acme/acct/1/order/1/finalize') self.orderr = messages.OrderResource( diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 5be458580..405fe7d9a 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -513,7 +513,6 @@ class Order(ResourceBody): """Order Resource Body. :ivar list of .Identifier: List of identifiers for the certificate. - :ivar acme.messages.Status status: :ivar list of str authorizations: URLs of authorizations. :ivar str certificate: URL to download certificate as a fullchain PEM. :ivar str finalize: URL to POST to to request issuance once all @@ -522,8 +521,6 @@ class Order(ResourceBody): :ivar .Error error: Any error that occurred during finalization, if applicable. """ identifiers = jose.Field('identifiers', omitempty=True) - status = jose.Field('status', decoder=Status.from_json, - omitempty=True, default=STATUS_PENDING) authorizations = jose.Field('authorizations', omitempty=True) certificate = jose.Field('certificate', omitempty=True) finalize = jose.Field('finalize', omitempty=True) @@ -553,4 +550,3 @@ class OrderResource(ResourceWithURI): class NewOrder(Order): """New order.""" resource_type = 'new-order' - resource = fields.Resource(resource_type) -- cgit v1.2.3 From 139ef206504ca241ae276b383c3cddee73db15cc Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 15 Oct 2018 10:41:04 -0700 Subject: Add debugging info for Nginx tls-sni and http integration tests purposes (#6414) --- certbot-nginx/certbot_nginx/http_01.py | 1 + certbot-nginx/certbot_nginx/tls_sni_01.py | 2 ++ certbot-nginx/tests/boulder-integration.sh | 1 + 3 files changed, 4 insertions(+) diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py index 677ce0737..9b385bc3b 100644 --- a/certbot-nginx/certbot_nginx/http_01.py +++ b/certbot-nginx/certbot_nginx/http_01.py @@ -102,6 +102,7 @@ class NginxHttp01(common.ChallengePerformer): config = [self._make_or_mod_server_block(achall) for achall in self.achalls] config = [x for x in config if x is not None] config = nginxparser.UnspacedList(config) + logger.debug("Generated server block:\n%s", str(config)) self.configurator.reverter.register_file_creation( True, self.challenge_conf) diff --git a/certbot-nginx/certbot_nginx/tls_sni_01.py b/certbot-nginx/certbot_nginx/tls_sni_01.py index 0fd37e0cb..d49ec8643 100644 --- a/certbot-nginx/certbot_nginx/tls_sni_01.py +++ b/certbot-nginx/certbot_nginx/tls_sni_01.py @@ -141,6 +141,8 @@ class NginxTlsSni01(common.TLSSNI01): with open(self.challenge_conf, "w") as new_conf: nginxparser.dump(config, new_conf) + logger.debug("Generated server block:\n%s", str(config)) + def _make_server_block(self, achall, addrs): """Creates a server block for a challenge. diff --git a/certbot-nginx/tests/boulder-integration.sh b/certbot-nginx/tests/boulder-integration.sh index 980b5d45a..194413f1d 100755 --- a/certbot-nginx/tests/boulder-integration.sh +++ b/certbot-nginx/tests/boulder-integration.sh @@ -35,6 +35,7 @@ test_deployment_and_rollback() { } export default_server="default_server" +nginx -v reload_nginx certbot_test_nginx --domains nginx.wtf run test_deployment_and_rollback nginx.wtf -- cgit v1.2.3 From 22da2447d5d89c11b9353ebbd7777690fee999df Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 17 Oct 2018 10:54:43 -0700 Subject: Stop caching the results of ipv6_info in http01.py (#6411) Stop caching the results of ipv6_info in http01.py. A call to choose_vhosts might change the ipv6 results of later calls. Add tests for this and default_listen_addresses more broadly. --- CHANGELOG.md | 1 + certbot-nginx/certbot_nginx/http_01.py | 6 +--- certbot-nginx/certbot_nginx/tests/http_01_test.py | 36 +++++++++++++++++++++++ certbot-nginx/certbot_nginx/tls_sni_01.py | 6 ++-- 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index def17accc..81cc12b15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Match Nginx parser update in allowing variable names to start with `${`. * Correct OVH integration tests on machines without internet access. +* Stop caching the results of ipv6_info in http01.py ## 0.27.1 - 2018-09-06 diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py index 9b385bc3b..e46d7b9b9 100644 --- a/certbot-nginx/certbot_nginx/http_01.py +++ b/certbot-nginx/certbot_nginx/http_01.py @@ -40,8 +40,6 @@ class NginxHttp01(common.ChallengePerformer): super(NginxHttp01, self).__init__(configurator) self.challenge_conf = os.path.join( configurator.config.config_dir, "le_http_01_cert_challenge.conf") - self._ipv6 = None - self._ipv6only = None def perform(self): """Perform a challenge on Nginx. @@ -121,9 +119,7 @@ class NginxHttp01(common.ChallengePerformer): self.configurator.config.http01_port) port = self.configurator.config.http01_port - if self._ipv6 is None or self._ipv6only is None: - self._ipv6, self._ipv6only = self.configurator.ipv6_info(port) - ipv6, ipv6only = self._ipv6, self._ipv6only + ipv6, ipv6only = self.configurator.ipv6_info(port) if ipv6: # If IPv6 is active in Nginx configuration diff --git a/certbot-nginx/certbot_nginx/tests/http_01_test.py b/certbot-nginx/certbot_nginx/tests/http_01_test.py index 0f764e92e..ed3c257ee 100644 --- a/certbot-nginx/certbot_nginx/tests/http_01_test.py +++ b/certbot-nginx/certbot_nginx/tests/http_01_test.py @@ -12,6 +12,7 @@ from certbot import achallenges from certbot.plugins import common_test from certbot.tests import acme_util +from certbot_nginx.obj import Addr from certbot_nginx.tests import util @@ -108,6 +109,41 @@ class HttpPerformTest(util.NginxTest): # self.assertEqual(vhost.addrs, set(v_addr2_print)) # self.assertEqual(vhost.names, set([response.z_domain.decode('ascii')])) + @mock.patch("certbot_nginx.configurator.NginxConfigurator.ipv6_info") + def test_default_listen_addresses_no_memoization(self, ipv6_info): + # pylint: disable=protected-access + ipv6_info.return_value = (True, True) + self.http01._default_listen_addresses() + self.assertEqual(ipv6_info.call_count, 1) + ipv6_info.return_value = (False, False) + self.http01._default_listen_addresses() + self.assertEqual(ipv6_info.call_count, 2) + + @mock.patch("certbot_nginx.configurator.NginxConfigurator.ipv6_info") + def test_default_listen_addresses_t_t(self, ipv6_info): + # pylint: disable=protected-access + ipv6_info.return_value = (True, True) + addrs = self.http01._default_listen_addresses() + http_addr = Addr.fromstring("80") + http_ipv6_addr = Addr.fromstring("[::]:80") + self.assertEqual(addrs, [http_addr, http_ipv6_addr]) + + @mock.patch("certbot_nginx.configurator.NginxConfigurator.ipv6_info") + def test_default_listen_addresses_t_f(self, ipv6_info): + # pylint: disable=protected-access + ipv6_info.return_value = (True, False) + addrs = self.http01._default_listen_addresses() + http_addr = Addr.fromstring("80") + http_ipv6_addr = Addr.fromstring("[::]:80 ipv6only=on") + self.assertEqual(addrs, [http_addr, http_ipv6_addr]) + + @mock.patch("certbot_nginx.configurator.NginxConfigurator.ipv6_info") + def test_default_listen_addresses_f_f(self, ipv6_info): + # pylint: disable=protected-access + ipv6_info.return_value = (False, False) + addrs = self.http01._default_listen_addresses() + http_addr = Addr.fromstring("80") + self.assertEqual(addrs, [http_addr]) if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-nginx/certbot_nginx/tls_sni_01.py b/certbot-nginx/certbot_nginx/tls_sni_01.py index d49ec8643..60ec1ed1a 100644 --- a/certbot-nginx/certbot_nginx/tls_sni_01.py +++ b/certbot-nginx/certbot_nginx/tls_sni_01.py @@ -51,9 +51,6 @@ class NginxTlsSni01(common.TLSSNI01): default_addr = "{0} ssl".format( self.configurator.config.tls_sni_01_port) - ipv6, ipv6only = self.configurator.ipv6_info( - self.configurator.config.tls_sni_01_port) - for achall in self.achalls: vhosts = self.configurator.choose_vhosts(achall.domain, create_if_no_match=True) @@ -61,6 +58,9 @@ class NginxTlsSni01(common.TLSSNI01): if vhosts and vhosts[0].addrs: addresses.append(list(vhosts[0].addrs)) else: + # choose_vhosts might have modified vhosts, so put this after + ipv6, ipv6only = self.configurator.ipv6_info( + self.configurator.config.tls_sni_01_port) if ipv6: # If IPv6 is active in Nginx configuration ipv6_addr = "[::]:{0} ssl".format( -- cgit v1.2.3 From 819f95c37d6d8471a5413137e033d30cea6822d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Wed, 17 Oct 2018 22:48:49 +0200 Subject: certbot_dns_linode: increase the default propagation interval (#6320) Using the default value of 16 minutes (960 seconds) for --dns-linode-propagation-seconds leads to DNS failures when the randomly selected Linode DNS is not the first one out of six, due to an additional delay before the other five are updated. The problem can be easily solved by increasing the wait interval, so this commit increases the default value to 20 minutes. More details: https://community.letsencrypt.org/t/dns-servers-used-by-letsencrypt-for-challenges/32127/16 --- certbot-dns-linode/certbot_dns_linode/__init__.py | 14 ++++++++++---- certbot-dns-linode/certbot_dns_linode/dns_linode.py | 2 +- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/certbot-dns-linode/certbot_dns_linode/__init__.py b/certbot-dns-linode/certbot_dns_linode/__init__.py index 0c445f45d..0a6ccec61 100644 --- a/certbot-dns-linode/certbot_dns_linode/__init__.py +++ b/certbot-dns-linode/certbot_dns_linode/__init__.py @@ -14,7 +14,11 @@ Named Arguments DNS to propagate before asking the ACME server to verify the DNS record. - (Default: 960) + (Default: 1200 because Linode + updates its first DNS every 15 + minutes and we allow 5 more minutes + for the update to reach the other 5 + servers) ========================================== =================================== @@ -74,13 +78,15 @@ Examples -d www.example.com .. code-block:: bash - :caption: To acquire a certificate for ``example.com``, waiting 60 seconds - for DNS propagation + :caption: To acquire a certificate for ``example.com``, waiting 1000 seconds + for DNS propagation (Linode updates its first DNS every 15 minutes + and we allow some extra time for the update to reach the other 5 + servers) certbot certonly \\ --dns-linode \\ --dns-linode-credentials ~/.secrets/certbot/linode.ini \\ - --dns-linode-propagation-seconds 60 \\ + --dns-linode-propagation-seconds 1000 \\ -d example.com """ diff --git a/certbot-dns-linode/certbot_dns_linode/dns_linode.py b/certbot-dns-linode/certbot_dns_linode/dns_linode.py index 323c0810a..cc29ce842 100644 --- a/certbot-dns-linode/certbot_dns_linode/dns_linode.py +++ b/certbot-dns-linode/certbot_dns_linode/dns_linode.py @@ -29,7 +29,7 @@ class Authenticator(dns_common.DNSAuthenticator): @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=960) + super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=1200) add('credentials', help='Linode credentials INI file.') def more_info(self): # pylint: disable=missing-docstring,no-self-use -- cgit v1.2.3 From 92501eaf8fe6afc514999b207037e5003f38c5db Mon Sep 17 00:00:00 2001 From: schoen Date: Wed, 17 Oct 2018 14:08:59 -0700 Subject: Note about running on web server, not PC (#6422) --- README.rst | 2 +- docs/install.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 0dbe1cdef..d75b44a65 100644 --- a/README.rst +++ b/README.rst @@ -6,7 +6,7 @@ Anyone who has gone through the trouble of setting up a secure website knows wha How you use Certbot depends on the configuration of your web server. The best way to get started is to use our `interactive guide `_. It generates instructions based on your configuration settings. In most cases, you’ll need `root or administrator access `_ to your web server to run Certbot. -If you’re using a hosted service and don’t have direct access to your web server, you might not be able to use Certbot. Check with your hosting provider for documentation about uploading certificates or using certificates issued by Let’s Encrypt. +Certbot is meant to be run directly on your web server, not on your personal computer. If you’re using a hosted service and don’t have direct access to your web server, you might not be able to use Certbot. Check with your hosting provider for documentation about uploading certificates or using certificates issued by Let’s Encrypt. Certbot is a fully-featured, extensible client for the Let's Encrypt CA (or any other CA that speaks the `ACME diff --git a/docs/install.rst b/docs/install.rst index f7504baa5..fc6abad7a 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -9,6 +9,8 @@ Get Certbot About Certbot ============= +*Certbot is meant to be run directly on a web server*, normally by a system administrator. In most cases, running Certbot on your personal computer is not a useful option. The instructions below relate to installing and running Certbot on a server. + Certbot is packaged for many common operating systems and web servers. Check whether ``certbot`` (or ``letsencrypt``) is packaged for your web server's OS by visiting certbot.eff.org_, where you will also find the correct installation instructions for -- cgit v1.2.3 From 3de3188dd6fe4c8c9952848c155a56944b55ec2d Mon Sep 17 00:00:00 2001 From: schoen Date: Thu, 18 Oct 2018 04:44:45 -0700 Subject: Warn manual authenticator users not to remove/undo previous challenges (#6370) * Warn users not to remove/undo previous challenges * Even more specific DNS challenge message * Fix spacing and variable names * Create a second test DNS challenge for UI testing * Changelog for subsequent manual challenge behavior --- CHANGELOG.md | 2 +- certbot/plugins/manual.py | 21 +++++++++++++++++++++ certbot/plugins/manual_test.py | 3 ++- certbot/tests/acme_util.py | 3 +++ 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81cc12b15..5dd51ef16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Changed -* +* `--manual` will explicitly warn users that earlier challenges should remain in place when setting up subsequent challenges. ### Fixed diff --git a/certbot/plugins/manual.py b/certbot/plugins/manual.py index 53533d35a..8723a1c62 100644 --- a/certbot/plugins/manual.py +++ b/certbot/plugins/manual.py @@ -94,6 +94,16 @@ using the secret key {key} when it receives a TLS ClientHello with the SNI extension set to {sni_domain} +""" + _SUBSEQUENT_CHALLENGE_INSTRUCTIONS = """ +(This must be set up in addition to the previous challenges; do not remove, +replace, or undo the previous challenge tasks yet.) +""" + _SUBSEQUENT_DNS_CHALLENGE_INSTRUCTIONS = """ +(This must be set up in addition to the previous challenges; do not remove, +replace, or undo the previous challenge tasks yet. Note that you might be +asked to create multiple distinct TXT records with the same name. This is +permitted by DNS standards.) """ def __init__(self, *args, **kwargs): @@ -103,6 +113,8 @@ when it receives a TLS ClientHello with the SNI extension set to self.env = dict() \ # type: Dict[achallenges.KeyAuthorizationAnnotatedChallenge, Dict[str, str]] self.tls_sni_01 = None + self.subsequent_dns_challenge = False + self.subsequent_any_challenge = False @classmethod def add_parser_arguments(cls, add): @@ -212,8 +224,17 @@ when it receives a TLS ClientHello with the SNI extension set to key=self.tls_sni_01.get_key_path(achall), port=self.config.tls_sni_01_port, sni_domain=self.tls_sni_01.get_z_domain(achall)) + if isinstance(achall.chall, challenges.DNS01): + if self.subsequent_dns_challenge: + # 2nd or later dns-01 challenge + msg += self._SUBSEQUENT_DNS_CHALLENGE_INSTRUCTIONS + self.subsequent_dns_challenge = True + elif self.subsequent_any_challenge: + # 2nd or later challenge of another type + msg += self._SUBSEQUENT_CHALLENGE_INSTRUCTIONS display = zope.component.getUtility(interfaces.IDisplay) display.notification(msg, wrap=False, force_interactive=True) + self.subsequent_any_challenge = True def cleanup(self, achalls): # pylint: disable=missing-docstring if self.conf('cleanup-hook'): diff --git a/certbot/plugins/manual_test.py b/certbot/plugins/manual_test.py index e5c22b377..ba4eb6889 100644 --- a/certbot/plugins/manual_test.py +++ b/certbot/plugins/manual_test.py @@ -20,8 +20,9 @@ class AuthenticatorTest(test_util.TempDirTestCase): super(AuthenticatorTest, self).setUp() self.http_achall = acme_util.HTTP01_A self.dns_achall = acme_util.DNS01_A + self.dns_achall_2 = acme_util.DNS01_A_2 self.tls_sni_achall = acme_util.TLSSNI01_A - self.achalls = [self.http_achall, self.dns_achall, self.tls_sni_achall] + self.achalls = [self.http_achall, self.dns_achall, self.tls_sni_achall, self.dns_achall_2] for d in ["config_dir", "work_dir", "in_progress"]: os.mkdir(os.path.join(self.tempdir, d)) # "backup_dir" and "temp_checkpoint_dir" get created in diff --git a/certbot/tests/acme_util.py b/certbot/tests/acme_util.py index 53a2f214a..2f9445694 100644 --- a/certbot/tests/acme_util.py +++ b/certbot/tests/acme_util.py @@ -21,6 +21,7 @@ HTTP01 = challenges.HTTP01( TLSSNI01 = challenges.TLSSNI01( token=jose.b64decode(b"evaGxfADs6pSRb2LAv9IZf17Dt3juxGJyPCt92wrDoA")) DNS01 = challenges.DNS01(token=b"17817c66b60ce2e4012dfad92657527a") +DNS01_2 = challenges.DNS01(token=b"cafecafecafecafecafecafe0feedbac") CHALLENGES = [HTTP01, TLSSNI01, DNS01] @@ -49,6 +50,7 @@ def chall_to_challb(chall, status): # pylint: disable=redefined-outer-name TLSSNI01_P = chall_to_challb(TLSSNI01, messages.STATUS_PENDING) HTTP01_P = chall_to_challb(HTTP01, messages.STATUS_PENDING) DNS01_P = chall_to_challb(DNS01, messages.STATUS_PENDING) +DNS01_P_2 = chall_to_challb(DNS01_2, messages.STATUS_PENDING) CHALLENGES_P = [HTTP01_P, TLSSNI01_P, DNS01_P] @@ -57,6 +59,7 @@ CHALLENGES_P = [HTTP01_P, TLSSNI01_P, DNS01_P] HTTP01_A = auth_handler.challb_to_achall(HTTP01_P, JWK, "example.com") TLSSNI01_A = auth_handler.challb_to_achall(TLSSNI01_P, JWK, "example.net") DNS01_A = auth_handler.challb_to_achall(DNS01_P, JWK, "example.org") +DNS01_A_2 = auth_handler.challb_to_achall(DNS01_P_2, JWK, "esimerkki.example.org") ACHALLENGES = [HTTP01_A, TLSSNI01_A, DNS01_A] -- cgit v1.2.3 From a3a3840e91d4ee086dd183400cbcd39ebd307938 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 18 Oct 2018 10:19:57 -0700 Subject: replace status field --- acme/acme/client_test.py | 1 + acme/acme/messages.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 965ece55d..4f8a1abe2 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -697,6 +697,7 @@ class ClientV2Test(ClientTestBase): self.order = messages.Order( identifiers=(self.authz.identifier, self.authz2.identifier), + status=messages.STATUS_PENDING, authorizations=(self.authzr.uri, self.authzr_uri2), finalize='https://www.letsencrypt-demo.org/acme/acct/1/order/1/finalize') self.orderr = messages.OrderResource( diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 405fe7d9a..df295bf2b 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -513,6 +513,7 @@ class Order(ResourceBody): """Order Resource Body. :ivar list of .Identifier: List of identifiers for the certificate. + :ivar acme.messages.Status status: :ivar list of str authorizations: URLs of authorizations. :ivar str certificate: URL to download certificate as a fullchain PEM. :ivar str finalize: URL to POST to to request issuance once all @@ -521,6 +522,8 @@ class Order(ResourceBody): :ivar .Error error: Any error that occurred during finalization, if applicable. """ identifiers = jose.Field('identifiers', omitempty=True) + status = jose.Field('status', decoder=Status.from_json, + omitempty=True, default=STATUS_PENDING) authorizations = jose.Field('authorizations', omitempty=True) certificate = jose.Field('certificate', omitempty=True) finalize = jose.Field('finalize', omitempty=True) -- cgit v1.2.3 From ee02ed65afe907a767e18b1536b5cde806a2cdd6 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 18 Oct 2018 10:26:37 -0700 Subject: remove default status from Order so that the status field isn't filled in upon boulder deserialization --- acme/acme/messages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acme/acme/messages.py b/acme/acme/messages.py index df295bf2b..7e86b0c3b 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -523,7 +523,7 @@ class Order(ResourceBody): """ identifiers = jose.Field('identifiers', omitempty=True) status = jose.Field('status', decoder=Status.from_json, - omitempty=True, default=STATUS_PENDING) + omitempty=True) authorizations = jose.Field('authorizations', omitempty=True) certificate = jose.Field('certificate', omitempty=True) finalize = jose.Field('finalize', omitempty=True) -- cgit v1.2.3 From 6500b9095e4e7b4d7b02cb39d7c78d322339dd11 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 18 Oct 2018 10:37:56 -0700 Subject: Add test to confirm that status isn't set on neworder object --- acme/acme/messages_test.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index 0e2d8c62d..876fbe825 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -424,6 +424,19 @@ class OrderResourceTest(unittest.TestCase): 'authorizations': None, }) +class NewOrderTest(unittest.TestCase): + """Tests for acme.messages.NewOrder.""" + + def setUp(self): + from acme.messages import NewOrder + self.reg = NewOrder( + identifiers=mock.sentinel.identifiers) + + def test_to_partial_json(self): + self.assertEqual(self.reg.to_json(), { + 'identifiers': mock.sentinel.identifiers, + }) + if __name__ == '__main__': unittest.main() # pragma: no cover -- cgit v1.2.3 From bfaf0296de4065a3b100580ff4f0b6190ebda800 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Thu, 18 Oct 2018 11:39:21 -0700 Subject: Also write README file to /etc/letsencrypt/live (#6377) We want to discourage people from moving things around in `/etc/letsencrypt/live`! So we dropped an extra README in the `/etc/` directory when it's first created. --- CHANGELOG.md | 3 ++- certbot/storage.py | 39 ++++++++++++++++++++++++--------------- certbot/tests/storage_test.py | 2 ++ 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5dd51ef16..20e82b5d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,8 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Changed -* `--manual` will explicitly warn users that earlier challenges should remain in place when setting up subsequent challenges. +* Write README to the base of (config-dir)/live directory +* `--manual` will explicitly warn users that earlier challenges should remain in place when setting up subsequent challenges. ### Fixed diff --git a/certbot/storage.py b/certbot/storage.py index 32d6771c2..c16ea35b8 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -214,6 +214,26 @@ def get_link_target(link): target = os.path.join(os.path.dirname(link), target) return os.path.abspath(target) +def _write_live_readme_to(readme_path, is_base_dir=False): + prefix = "" + if is_base_dir: + prefix = "[cert name]/" + with open(readme_path, "w") as f: + logger.debug("Writing README to %s.", readme_path) + f.write("This directory contains your keys and certificates.\n\n" + "`{prefix}privkey.pem` : the private key for your certificate.\n" + "`{prefix}fullchain.pem`: the certificate file used in most server software.\n" + "`{prefix}chain.pem` : used for OCSP stapling in Nginx >=1.3.7.\n" + "`{prefix}cert.pem` : will break many server configurations, and " + "should not be used\n" + " without reading further documentation (see link below).\n\n" + "WARNING: DO NOT MOVE OR RENAME THESE FILES!\n" + " Certbot expects these files to remain in this location in order\n" + " to function properly!\n\n" + "We recommend not moving these files. For more information, see the Certbot\n" + "User Guide at https://certbot.eff.org/docs/using.html#where-are-my-" + "certificates.\n".format(prefix=prefix)) + def _relevant(option): """ @@ -1003,6 +1023,9 @@ class RenewableCert(object): logger.debug("Creating directory %s.", i) config_file, config_filename = util.unique_lineage_name( cli_config.renewal_configs_dir, lineagename) + base_readme_path = os.path.join(cli_config.live_dir, README) + if not os.path.exists(base_readme_path): + _write_live_readme_to(base_readme_path, is_base_dir=True) # Determine where on disk everything will go # lineagename will now potentially be modified based on which @@ -1045,21 +1068,7 @@ class RenewableCert(object): # Write a README file to the live directory readme_path = os.path.join(live_dir, README) - with open(readme_path, "w") as f: - logger.debug("Writing README to %s.", readme_path) - f.write("This directory contains your keys and certificates.\n\n" - "`privkey.pem` : the private key for your certificate.\n" - "`fullchain.pem`: the certificate file used in most server software.\n" - "`chain.pem` : used for OCSP stapling in Nginx >=1.3.7.\n" - "`cert.pem` : will break many server configurations, and " - "should not be used\n" - " without reading further documentation (see link below).\n\n" - "WARNING: DO NOT MOVE THESE FILES!\n" - " Certbot expects these files to remain in this location in order\n" - " to function properly!\n\n" - "We recommend not moving these files. For more information, see the Certbot\n" - "User Guide at https://certbot.eff.org/docs/using.html#where-are-my-" - "certificates.\n") + _write_live_readme_to(readme_path) # Document what we've done in a new renewal config file config_file.close() diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 53a976f8d..078a2858f 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -625,6 +625,8 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertTrue(result._consistent()) self.assertTrue(os.path.exists(os.path.join( self.config.renewal_configs_dir, "the-lineage.com.conf"))) + self.assertTrue(os.path.exists(os.path.join( + self.config.live_dir, "README"))) self.assertTrue(os.path.exists(os.path.join( self.config.live_dir, "the-lineage.com", "README"))) with open(result.fullchain, "rb") as f: -- cgit v1.2.3 From 0dab41ee139a9186cfb82707981acd56b60bf095 Mon Sep 17 00:00:00 2001 From: Daniel McCarney Date: Thu, 18 Oct 2018 16:12:47 -0400 Subject: docs: remove mentions of #letsencrypt on Freenode. (#6419) * docs: remove mentions of #letsencrypt on Freenode. * docs: remove unused Freenode link --- CHANGELOG.md | 1 + README.rst | 4 ---- docs/using.rst | 3 --- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20e82b5d7..5060a7038 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Changed +* Removed documentation mentions of `#letsencrypt` IRC on Freenode. * Write README to the base of (config-dir)/live directory * `--manual` will explicitly warn users that earlier challenges should remain in place when setting up subsequent challenges. diff --git a/README.rst b/README.rst index d75b44a65..62681c7eb 100644 --- a/README.rst +++ b/README.rst @@ -91,8 +91,6 @@ Main Website: https://certbot.eff.org Let's Encrypt Website: https://letsencrypt.org -IRC Channel: #letsencrypt on `Freenode`_ - Community: https://community.letsencrypt.org ACME spec: http://ietf-wg-acme.github.io/acme/ @@ -101,8 +99,6 @@ ACME working area in github: https://github.com/ietf-wg-acme/acme |build-status| |coverage| |docs| |container| -.. _Freenode: https://webchat.freenode.net?channels=%23letsencrypt - .. |build-status| image:: https://travis-ci.org/certbot/certbot.svg?branch=master :target: https://travis-ci.org/certbot/certbot :alt: Travis CI status diff --git a/docs/using.rst b/docs/using.rst index 2f45feca9..1fa13e022 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -988,9 +988,6 @@ Getting help If you're having problems, we recommend posting on the Let's Encrypt `Community Forum `_. -You can also chat with us on IRC: `(#letsencrypt @ -freenode) `_ - If you find a bug in the software, please do report it in our `issue tracker `_. Remember to give us as much information as possible: -- cgit v1.2.3 From 8dd68a655104db2d642e65f48738b92bf133e709 Mon Sep 17 00:00:00 2001 From: sydneyli Date: Fri, 19 Oct 2018 12:30:32 -0700 Subject: Add and test new nginx parsing abstractions (#6383) * feat(nginx): add and test new parsing abstractions * chore(nginx parser): fix mypy and address small comments * chore(nginx parser): clean up by removing context object * fix integration test and lint --- certbot-nginx/certbot_nginx/parser_obj.py | 392 +++++++++++++++++++++ .../certbot_nginx/tests/parser_obj_test.py | 253 +++++++++++++ tests/integration/_common.sh | 1 + 3 files changed, 646 insertions(+) create mode 100644 certbot-nginx/certbot_nginx/parser_obj.py create mode 100644 certbot-nginx/certbot_nginx/tests/parser_obj_test.py diff --git a/certbot-nginx/certbot_nginx/parser_obj.py b/certbot-nginx/certbot_nginx/parser_obj.py new file mode 100644 index 000000000..f01cb2fd3 --- /dev/null +++ b/certbot-nginx/certbot_nginx/parser_obj.py @@ -0,0 +1,392 @@ +""" This file contains parsing routines and object classes to help derive meaning from +raw lists of tokens from pyparsing. """ + +import abc +import logging +import six + +from certbot import errors + +from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module + +logger = logging.getLogger(__name__) +COMMENT = " managed by Certbot" +COMMENT_BLOCK = ["#", COMMENT] + +class Parsable(object): + """ Abstract base class for "Parsable" objects whose underlying representation + is a tree of lists. + + :param .Parsable parent: This object's parsed parent in the tree + """ + + __metaclass__ = abc.ABCMeta + + def __init__(self, parent=None): + self._data = [] # type: List[object] + self._tabs = None + self.parent = parent + + @classmethod + def parsing_hooks(cls): + """Returns object types that this class should be able to `parse` recusrively. + The order of the objects indicates the order in which the parser should + try to parse each subitem. + :returns: A list of Parsable classes. + :rtype list: + """ + return (Block, Sentence, Statements) + + @staticmethod + @abc.abstractmethod + def should_parse(lists): + """ Returns whether the contents of `lists` can be parsed into this object. + + :returns: Whether `lists` can be parsed as this object. + :rtype bool: + """ + raise NotImplementedError() + + @abc.abstractmethod + def parse(self, raw_list, add_spaces=False): + """ Loads information into this object from underlying raw_list structure. + Each Parsable object might make different assumptions about the structure of + raw_list. + + :param list raw_list: A list or sublist of tokens from pyparsing, containing whitespace + as separate tokens. + :param bool add_spaces: If set, the method can and should manipulate and insert spacing + between non-whitespace tokens and lists to delimit them. + :raises .errors.MisconfigurationError: when the assumptions about the structure of + raw_list are not met. + """ + raise NotImplementedError() + + @abc.abstractmethod + def iterate(self, expanded=False, match=None): + """ Iterates across this object. If this object is a leaf object, only yields + itself. If it contains references other parsing objects, and `expanded` is set, + this function should first yield itself, then recursively iterate across all of them. + :param bool expanded: Whether to recursively iterate on possible children. + :param callable match: If provided, an object is only iterated if this callable + returns True when called on that object. + + :returns: Iterator over desired objects. + """ + raise NotImplementedError() + + @abc.abstractmethod + def get_tabs(self): + """ Guess at the tabbing style of this parsed object, based on whitespace. + + If this object is a leaf, it deducts the tabbing based on its own contents. + Other objects may guess by calling `get_tabs` recursively on child objects. + + :returns: Guess at tabbing for this object. Should only return whitespace strings + that does not contain newlines. + :rtype str: + """ + raise NotImplementedError() + + @abc.abstractmethod + def set_tabs(self, tabs=" "): + """This tries to set and alter the tabbing of the current object to a desired + whitespace string. Primarily meant for objects that were constructed, so they + can conform to surrounding whitespace. + + :param str tabs: A whitespace string (not containing newlines). + """ + raise NotImplementedError() + + def dump(self, include_spaces=False): + """ Dumps back to pyparsing-like list tree. The opposite of `parse`. + + Note: if this object has not been modified, `dump` with `include_spaces=True` + should always return the original input of `parse`. + + :param bool include_spaces: If set to False, magically hides whitespace tokens from + dumped output. + + :returns: Pyparsing-like list tree. + :rtype list: + """ + return [elem.dump(include_spaces) for elem in self._data] + +class Statements(Parsable): + """ A group or list of "Statements". A Statement is either a Block or a Sentence. + + The underlying representation is simply a list of these Statement objects, with + an extra `_trailing_whitespace` string to keep track of the whitespace that does not + precede any more statements. + """ + def __init__(self, parent=None): + super(Statements, self).__init__(parent) + self._trailing_whitespace = None + + # ======== Begin overridden functions + + @staticmethod + def should_parse(lists): + return isinstance(lists, list) + + def set_tabs(self, tabs=" "): + """ Sets the tabbing for this set of statements. Does this by calling `set_tabs` + on each of the child statements. + + Then, if a parent is present, sets trailing whitespace to parent tabbing. This + is so that the trailing } of any Block that contains Statements lines up + with parent tabbing. + """ + for statement in self._data: + statement.set_tabs(tabs) + if self.parent is not None: + self._trailing_whitespace = "\n" + self.parent.get_tabs() + + def parse(self, parse_this, add_spaces=False): + """ Parses a list of statements. + Expects all elements in `parse_this` to be parseable by `type(self).parsing_hooks`, + with an optional whitespace string at the last index of `parse_this`. + """ + if not isinstance(parse_this, list): + raise errors.MisconfigurationError("Statements parsing expects a list!") + # If there's a trailing whitespace in the list of statements, keep track of it. + if len(parse_this) > 0 and isinstance(parse_this[-1], six.string_types) \ + and parse_this[-1].isspace(): + self._trailing_whitespace = parse_this[-1] + parse_this = parse_this[:-1] + self._data = [parse_raw(elem, self, add_spaces) for elem in parse_this] + + def get_tabs(self): + """ Takes a guess at the tabbing of all contained Statements by retrieving the + tabbing of the first Statement.""" + if len(self._data) > 0: + return self._data[0].get_tabs() + return "" + + def dump(self, include_spaces=False): + """ Dumps this object by first dumping each statement, then appending its + trailing whitespace (if `include_spaces` is set) """ + data = super(Statements, self).dump(include_spaces) + if include_spaces and self._trailing_whitespace is not None: + return data + [self._trailing_whitespace] + return data + + def iterate(self, expanded=False, match=None): + """ Combines each statement's iterator. """ + for elem in self._data: + for sub_elem in elem.iterate(expanded, match): + yield sub_elem + + # ======== End overridden functions + +def _space_list(list_): + """ Inserts whitespace between adjacent non-whitespace tokens. """ + spaced_statement = [] # type: List[str] + for i in reversed(six.moves.xrange(len(list_))): + spaced_statement.insert(0, list_[i]) + if i > 0 and not list_[i].isspace() and not list_[i-1].isspace(): + spaced_statement.insert(0, " ") + return spaced_statement + +class Sentence(Parsable): + """ A list of words. Non-whitespace words are typically separated with whitespace tokens. """ + + # ======== Begin overridden functions + + @staticmethod + def should_parse(lists): + """ Returns True if `lists` can be parseable as a `Sentence`-- that is, + every element is a string type. + + :param list lists: The raw unparsed list to check. + + :returns: whether this lists is parseable by `Sentence`. + """ + return isinstance(lists, list) and len(lists) > 0 and \ + all([isinstance(elem, six.string_types) for elem in lists]) + + def parse(self, parse_this, add_spaces=False): + """ Parses a list of string types into this object. + If add_spaces is set, adds whitespace tokens between adjacent non-whitespace tokens.""" + if add_spaces: + parse_this = _space_list(parse_this) + if not isinstance(parse_this, list) or \ + any([not isinstance(elem, six.string_types) for elem in parse_this]): + raise errors.MisconfigurationError("Sentence parsing expects a list of string types.") + self._data = parse_this + + def iterate(self, expanded=False, match=None): + """ Simply yields itself. """ + if match is None or match(self): + yield self + + def set_tabs(self, tabs=" "): + """ Sets the tabbing on this sentence. Inserts a newline and `tabs` at the + beginning of `self._data`. """ + if self._data[0].isspace(): + return + self._data.insert(0, "\n" + tabs) + + def dump(self, include_spaces=False): + """ Dumps this sentence. If include_spaces is set, includes whitespace tokens.""" + if not include_spaces: + return self.words + return self._data + + def get_tabs(self): + """ Guesses at the tabbing of this sentence. If the first element is whitespace, + returns the whitespace after the rightmost newline in the string. """ + first = self._data[0] + if not first.isspace(): + return "" + rindex = first.rfind("\n") + return first[rindex+1:] + + # ======== End overridden functions + + @property + def words(self): + """ Iterates over words, but without spaces. Like Unspaced List. """ + return [word.strip("\"\'") for word in self._data if not word.isspace()] + + def __getitem__(self, index): + return self.words[index] + + def __contains__(self, word): + return word in self.words + +class Block(Parsable): + """ Any sort of bloc, denoted by a block name and curly braces, like so: + The parsed block: + block name { + content 1; + content 2; + } + might be represented with the list [names, contents], where + names = ["block", " ", "name", " "] + contents = [["\n ", "content", " ", "1"], ["\n ", "content", " ", "2"], "\n"] + """ + def __init__(self, parent=None): + super(Block, self).__init__(parent) + self.names = None # type: Sentence + self.contents = None # type: Block + + @staticmethod + def should_parse(lists): + """ Returns True if `lists` can be parseable as a `Block`-- that is, + it's got a length of 2, the first element is a `Sentence` and the second can be + a `Statements`. + + :param list lists: The raw unparsed list to check. + + :returns: whether this lists is parseable by `Block`. """ + return isinstance(lists, list) and len(lists) == 2 and \ + Sentence.should_parse(lists[0]) and isinstance(lists[1], list) + + def set_tabs(self, tabs=" "): + """ Sets tabs by setting equivalent tabbing on names, then adding tabbing + to contents.""" + self.names.set_tabs(tabs) + self.contents.set_tabs(tabs + " ") + + def iterate(self, expanded=False, match=None): + """ Iterator over self, and if expanded is set, over its contents. """ + if match is None or match(self): + yield self + if expanded: + for elem in self.contents.iterate(expanded, match): + yield elem + + def parse(self, parse_this, add_spaces=False): + """ Parses a list that resembles a block. + + The assumptions that this routine makes are: + 1. the first element of `parse_this` is a valid Sentence. + 2. the second element of `parse_this` is a valid Statement. + If add_spaces is set, we call it recursively on `names` and `contents`, and + add an extra trailing space to `names` (to separate the block's opening bracket + and the block name). + """ + if not Block.should_parse(parse_this): + raise errors.MisconfigurationError("Block parsing expects a list of length 2. " + "First element should be a list of string types (the bloc names), " + "and second should be another list of statements (the bloc content).") + self.names = Sentence(self) + if add_spaces: + parse_this[0].append(" ") + self.names.parse(parse_this[0], add_spaces) + self.contents = Statements(self) + self.contents.parse(parse_this[1], add_spaces) + self._data = [self.names, self.contents] + + def get_tabs(self): + """ Guesses tabbing by retrieving tabbing guess of self.names. """ + return self.names.get_tabs() + +def _is_comment(parsed_obj): + """ Checks whether parsed_obj is a comment. + + :param .Parsable parsed_obj: + + :returns: whether parsed_obj represents a comment sentence. + :rtype bool: + """ + if not isinstance(parsed_obj, Sentence): + return False + return parsed_obj.words[0] == "#" + +def _is_certbot_comment(parsed_obj): + """ Checks whether parsed_obj is a "managed by Certbot" comment. + + :param .Parsable parsed_obj: + + :returns: whether parsed_obj is a "managed by Certbot" comment. + :rtype bool: + """ + if not _is_comment(parsed_obj): + return False + if len(parsed_obj.words) != len(COMMENT_BLOCK): + return False + for i, word in enumerate(parsed_obj.words): + if word != COMMENT_BLOCK[i]: + return False + return True + +def _certbot_comment(parent, preceding_spaces=4): + """ A "Managed by Certbot" comment. + :param int preceding_spaces: Number of spaces between the end of the previous + statement and the comment. + :returns: Sentence containing the comment. + :rtype: .Sentence + """ + result = Sentence(parent) + result.parse([" " * preceding_spaces] + COMMENT_BLOCK) + return result + +def _choose_parser(parent, list_): + """ Choose a parser from type(parent).parsing_hooks, depending on whichever hook + returns True first. """ + hooks = Parsable.parsing_hooks() + if parent: + hooks = type(parent).parsing_hooks() + for type_ in hooks: + if type_.should_parse(list_): + return type_(parent) + raise errors.MisconfigurationError( + "None of the parsing hooks succeeded, so we don't know how to parse this set of lists.") + +def parse_raw(lists_, parent=None, add_spaces=False): + """ Primary parsing factory function. + + :param list lists_: raw lists from pyparsing to parse. + :param .Parent parent: The parent containing this object. + :param bool add_spaces: Whether to pass add_spaces to the parser. + + :returns .Parsable: The parsed object. + + :raises errors.MisconfigurationError: If no parsing hook passes, and we can't + determine which type to parse the raw lists into. + """ + parser = _choose_parser(parent, lists_) + parser.parse(lists_, add_spaces) + return parser diff --git a/certbot-nginx/certbot_nginx/tests/parser_obj_test.py b/certbot-nginx/certbot_nginx/tests/parser_obj_test.py new file mode 100644 index 000000000..c9c9dd440 --- /dev/null +++ b/certbot-nginx/certbot_nginx/tests/parser_obj_test.py @@ -0,0 +1,253 @@ +""" Tests for functions and classes in parser_obj.py """ + +import unittest +import mock + +from certbot_nginx.parser_obj import parse_raw +from certbot_nginx.parser_obj import COMMENT_BLOCK + +class CommentHelpersTest(unittest.TestCase): + def test_is_comment(self): + from certbot_nginx.parser_obj import _is_comment + self.assertTrue(_is_comment(parse_raw(['#']))) + self.assertTrue(_is_comment(parse_raw(['#', ' literally anything else']))) + self.assertFalse(_is_comment(parse_raw(['not', 'even', 'a', 'comment']))) + + def test_is_certbot_comment(self): + from certbot_nginx.parser_obj import _is_certbot_comment + self.assertTrue(_is_certbot_comment( + parse_raw(COMMENT_BLOCK))) + self.assertFalse(_is_certbot_comment( + parse_raw(['#', ' not a certbot comment']))) + self.assertFalse(_is_certbot_comment( + parse_raw(['#', ' managed by Certbot', ' also not a certbot comment']))) + self.assertFalse(_is_certbot_comment( + parse_raw(['not', 'even', 'a', 'comment']))) + + def test_certbot_comment(self): + from certbot_nginx.parser_obj import _certbot_comment, _is_certbot_comment + comment = _certbot_comment(None) + self.assertTrue(_is_certbot_comment(comment)) + self.assertEqual(comment.dump(), COMMENT_BLOCK) + self.assertEqual(comment.dump(True), [' '] + COMMENT_BLOCK) + self.assertEqual(_certbot_comment(None, 2).dump(True), + [' '] + COMMENT_BLOCK) + +class ParsingHooksTest(unittest.TestCase): + def test_is_sentence(self): + from certbot_nginx.parser_obj import Sentence + self.assertFalse(Sentence.should_parse([])) + self.assertTrue(Sentence.should_parse([''])) + self.assertTrue(Sentence.should_parse(['word'])) + self.assertTrue(Sentence.should_parse(['two', 'words'])) + self.assertFalse(Sentence.should_parse([[]])) + self.assertFalse(Sentence.should_parse(['word', []])) + + def test_is_block(self): + from certbot_nginx.parser_obj import Block + self.assertFalse(Block.should_parse([])) + self.assertFalse(Block.should_parse([''])) + self.assertFalse(Block.should_parse(['two', 'words'])) + self.assertFalse(Block.should_parse([[[]], []])) + self.assertFalse(Block.should_parse([['block_name'], ['hi', []], []])) + self.assertFalse(Block.should_parse([['block_name'], 'lol'])) + self.assertTrue(Block.should_parse([['block_name'], ['hi', []]])) + self.assertTrue(Block.should_parse([['hello'], []])) + self.assertTrue(Block.should_parse([['block_name'], [['many'], ['statements'], 'here']])) + self.assertTrue(Block.should_parse([['if', ' ', '(whatever)'], ['hi']])) + + def test_parse_raw(self): + fake_parser1 = mock.Mock() + fake_parser1.should_parse = lambda x: True + fake_parser2 = mock.Mock() + fake_parser2.should_parse = lambda x: False + # First encountered "match" should parse. + parse_raw([]) + fake_parser1.called_once() + fake_parser2.not_called() + fake_parser1.reset_mock() + # "match" that returns False shouldn't parse. + parse_raw([]) + fake_parser1.not_called() + fake_parser2.called_once() + + @mock.patch("certbot_nginx.parser_obj.Parsable.parsing_hooks") + def test_parse_raw_no_match(self, parsing_hooks): + from certbot import errors + fake_parser1 = mock.Mock() + fake_parser1.should_parse = lambda x: False + parsing_hooks.return_value = (fake_parser1,) + self.assertRaises(errors.MisconfigurationError, parse_raw, []) + parsing_hooks.return_value = tuple() + self.assertRaises(errors.MisconfigurationError, parse_raw, []) + + def test_parse_raw_passes_add_spaces(self): + fake_parser1 = mock.Mock() + fake_parser1.should_parse = lambda x: True + parse_raw([]) + fake_parser1.parse.called_with([None]) + parse_raw([], add_spaces=True) + fake_parser1.parse.called_with([None, True]) + +class SentenceTest(unittest.TestCase): + def setUp(self): + from certbot_nginx.parser_obj import Sentence + self.sentence = Sentence(None) + + def test_parse_bad_sentence_raises_error(self): + from certbot import errors + self.assertRaises(errors.MisconfigurationError, self.sentence.parse, 'lol') + self.assertRaises(errors.MisconfigurationError, self.sentence.parse, [[]]) + self.assertRaises(errors.MisconfigurationError, self.sentence.parse, [5]) + + def test_parse_sentence_words_hides_spaces(self): + og_sentence = ['\r\n', 'hello', ' ', ' ', '\t\n ', 'lol', ' ', 'spaces'] + self.sentence.parse(og_sentence) + self.assertEquals(self.sentence.words, ['hello', 'lol', 'spaces']) + self.assertEquals(self.sentence.dump(), ['hello', 'lol', 'spaces']) + self.assertEquals(self.sentence.dump(True), og_sentence) + + def test_parse_sentence_with_add_spaces(self): + self.sentence.parse(['hi', 'there'], add_spaces=True) + self.assertEquals(self.sentence.dump(True), ['hi', ' ', 'there']) + self.sentence.parse(['one', ' ', 'space', 'none'], add_spaces=True) + self.assertEquals(self.sentence.dump(True), ['one', ' ', 'space', ' ', 'none']) + + def test_iterate(self): + expected = [['1', '2', '3']] + self.sentence.parse(['1', ' ', '2', ' ', '3']) + for i, sentence in enumerate(self.sentence.iterate()): + self.assertEquals(sentence.dump(), expected[i]) + + def test_set_tabs(self): + self.sentence.parse(['tabs', 'pls'], add_spaces=True) + self.sentence.set_tabs() + self.assertEquals(self.sentence.dump(True)[0], '\n ') + self.sentence.parse(['tabs', 'pls'], add_spaces=True) + + def test_get_tabs(self): + self.sentence.parse(['no', 'tabs']) + self.assertEquals(self.sentence.get_tabs(), '') + self.sentence.parse(['\n \n ', 'tabs']) + self.assertEquals(self.sentence.get_tabs(), ' ') + self.sentence.parse(['\n\t ', 'tabs']) + self.assertEquals(self.sentence.get_tabs(), '\t ') + self.sentence.parse(['\n\t \n', 'tabs']) + self.assertEquals(self.sentence.get_tabs(), '') + +class BlockTest(unittest.TestCase): + def setUp(self): + from certbot_nginx.parser_obj import Block + self.bloc = Block(None) + self.name = ['server', 'name'] + self.contents = [['thing', '1'], ['thing', '2'], ['another', 'one']] + self.bloc.parse([self.name, self.contents]) + + def test_iterate(self): + # Iterates itself normally + self.assertEquals(self.bloc, next(self.bloc.iterate())) + # Iterates contents while expanded + expected = [self.bloc.dump()] + self.contents + for i, elem in enumerate(self.bloc.iterate(expanded=True)): + self.assertEquals(expected[i], elem.dump()) + + def test_iterate_match(self): + # can match on contents while expanded + from certbot_nginx.parser_obj import Block, Sentence + expected = [['thing', '1'], ['thing', '2']] + for i, elem in enumerate(self.bloc.iterate(expanded=True, + match=lambda x: isinstance(x, Sentence) and 'thing' in x.words)): + self.assertEquals(expected[i], elem.dump()) + # can match on self + self.assertEquals(self.bloc, next(self.bloc.iterate( + expanded=True, + match=lambda x: isinstance(x, Block) and 'server' in x.names))) + + def test_parse_with_added_spaces(self): + import copy + self.bloc.parse([copy.copy(self.name), self.contents], add_spaces=True) + self.assertEquals(self.bloc.dump(), [self.name, self.contents]) + self.assertEquals(self.bloc.dump(True), [ + ['server', ' ', 'name', ' '], + [['thing', ' ', '1'], + ['thing', ' ', '2'], + ['another', ' ', 'one']]]) + + def test_bad_parse_raises_error(self): + from certbot import errors + self.assertRaises(errors.MisconfigurationError, self.bloc.parse, [[[]], [[]]]) + self.assertRaises(errors.MisconfigurationError, self.bloc.parse, ['lol']) + self.assertRaises(errors.MisconfigurationError, self.bloc.parse, ['fake', 'news']) + + def test_set_tabs(self): + self.bloc.set_tabs() + self.assertEquals(self.bloc.names.dump(True)[0], '\n ') + for elem in self.bloc.contents.dump(True)[:-1]: + self.assertEquals(elem[0], '\n ') + self.assertEquals(self.bloc.contents.dump(True)[-1][0], '\n') + + def test_get_tabs(self): + self.bloc.parse([[' \n \t', 'lol'], []]) + self.assertEquals(self.bloc.get_tabs(), ' \t') + +class StatementsTest(unittest.TestCase): + def setUp(self): + from certbot_nginx.parser_obj import Statements + self.statements = Statements(None) + self.raw = [ + ['sentence', 'one'], + ['sentence', 'two'], + ['and', 'another'] + ] + self.raw_spaced = [ + ['\n ', 'sentence', ' ', 'one'], + ['\n ', 'sentence', ' ', 'two'], + ['\n ', 'and', ' ', 'another'], + '\n\n' + ] + + def test_set_tabs(self): + self.statements.parse(self.raw) + self.statements.set_tabs() + for statement in self.statements.iterate(): + self.assertEquals(statement.dump(True)[0], '\n ') + + def test_set_tabs_with_parent(self): + # Trailing whitespace should inherit from parent tabbing. + self.statements.parse(self.raw) + self.statements.parent = mock.Mock() + self.statements.parent.get_tabs.return_value = '\t\t' + self.statements.set_tabs() + for statement in self.statements.iterate(): + self.assertEquals(statement.dump(True)[0], '\n ') + self.assertEquals(self.statements.dump(True)[-1], '\n\t\t') + + def test_get_tabs(self): + self.raw[0].insert(0, '\n \n \t') + self.statements.parse(self.raw) + self.assertEquals(self.statements.get_tabs(), ' \t') + self.statements.parse([]) + self.assertEquals(self.statements.get_tabs(), '') + + def test_parse_with_added_spaces(self): + self.statements.parse(self.raw, add_spaces=True) + self.assertEquals(self.statements.dump(True)[0], ['sentence', ' ', 'one']) + + def test_parse_bad_list_raises_error(self): + from certbot import errors + self.assertRaises(errors.MisconfigurationError, self.statements.parse, 'lol not a list') + + def test_parse_hides_trailing_whitespace(self): + self.statements.parse(self.raw + ['\n\n ']) + self.assertTrue(isinstance(self.statements.dump()[-1], list)) + self.assertTrue(self.statements.dump(True)[-1].isspace()) + self.assertEquals(self.statements.dump(True)[-1], '\n\n ') + + def test_iterate(self): + self.statements.parse(self.raw) + expected = [['sentence', 'one'], ['sentence', 'two']] + for i, elem in enumerate(self.statements.iterate(match=lambda x: 'sentence' in x)): + self.assertEquals(expected[i], elem.dump()) + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index a8d35ed89..1e444fa26 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -25,6 +25,7 @@ certbot_test_no_force_renew () { omit_patterns="*/*.egg-info/*,*/dns_common*,*/setup.py,*/test_*,*/tests/*" omit_patterns="$omit_patterns,*_test.py,*_test_*,certbot-apache/*" omit_patterns="$omit_patterns,certbot-compatibility-test/*,certbot-dns*/" + omit_patterns="$omit_patterns,certbot-nginx/certbot_nginx/parser_obj.py" coverage run \ --append \ --source $sources \ -- cgit v1.2.3 From 1e8c13ebf905d1972e3c3c2474650520df1be921 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 19 Oct 2018 23:53:15 +0200 Subject: [Windows] Create the CI logic (#6374) So here we are: after #6361 has been merged, time is to provide an environment to execute the automated testing on Windows. Here are the assertions used to build the CI on Windows: every test running on Linux should ultimately be runnable on Windows, in a cross-platform compatible manner (there is one or two exception, when a test does not have any meaning for Windows), currently some tests are not runnable on Windows: theses tests are ignored by default when the environment is Windows using a custom decorator: @broken_on_windows, test environment should have functionalities similar to Travis, in particular an execution test matrix against various versions of Python and Windows, so test execution is done through AppVeyor, as it supports the requirements: it add a CI step along Travis and Codecov for each PR, all of this ensuring that Certbot is entirely functional on both Linux and Windows, code in tests can be changed, but code in Certbot should be changed as little as possible, to avoid regression risks. So far in this PR, I focused on the tests on Certbot core and ACME library. Concerning the plugins, it will be done later, for plugins which have an interest on Windows. Test are executed against Python 3.4, 3.5, 3.6 and 3.7, for Windows Server 2012 R2 and Windows Server 2016. I succeeded at making 258/259 of acme tests to work, and 828/868 of certbot core tests to work. Most of the errors where not because of Certbot itself, but because of how the tests are written. After redesigning some test utilitaries, and things like file path handling, or CRLF/LF, a lot of the errors vanished. I needed also to ignore a lot of IO errors typically occurring when a tearDown test process tries to delete a file before it has been closed: this kind of behavior is acceptable for Linux, but not for Windows. As a consequence, and until the tearDown process is improved, a lot of temporary files are not cleared on Windows after a test campaign. Remaining broken tests requires a more subtile approach to solve the errors, I will correct them progressively in future PR. Last words about tox. I did not used the existing tox.ini for now. It is just to far from what is supported on Windows: lot of bash scripts that should be rewritten completely, and that contain test logic not ready/relevant for Windows (plugin tests, Docker compilation/test, GNU distribution versatility handling and so on). So I use an independent file tox-win.ini for now, with the goal to merge it ultimately with the existing logic. * Define a tox configuration for windows, to execute tests against Python 3.4, 3.5, 3.6 and 3.7 + code coverage on Codecov.io * Correct windows compatibility on certbot codebase * Correct windows compatibility on certbot display functionalities * Correct windows compatibility on certbot plugins * Correct test utils to run tests on windows. Add decorator to skip (permanently) or mark broken (temporarily) tests on windows * Correct tests on certbot core to run them both on windows and linux. Mark some of them as broken on windows for now. * Lock tests are completely skipped on windows. Planned to be replace in next PR. * Correct tests on certbot display to run them both on windows and linux. Mark some of them as broken on windows for now. * Correct test utils for acme on windows. Add decorator to skip (permanently) or mark broken (temporarily) tests on windows. * Correct acme tests to run them both on windows and linux. Allow a reduction of code coverage of 1% on acme code base. * Create AppVeyor CI for Certbot on Windows, to run the test matrix (py34,35,36,37+coverage) on Windows Server 2012 R2 and Windows Server 2016. * Update changelog with Windows compatibility of Certbot. * Corrections about tox, pyreadline and CI logic * Correct english * Some corrections for acme * Newlines corrections * Remove changelog * Use os.devnull instead of /dev/null to be used on Windows * Uid is a always a number now. * Correct linting * PR https://github.com/python/typeshed/pull/2136 has been merge to third-party upstream 6 months ago, so code patch can be removed. * And so acme coverage should be 100% again. * More compatible tests Windows+Linux * Use stable line separator * Remove unused import * Do not rely on pytest in certbot tests * Use json.dumps to another json embedding weird characters * Change comment * Add import * Test rolling builds #1 * Test rolling builds #2 * Correction on json serialization * It seems that rolling builds are not canceling jobs on PR. Revert back to fail fast code in the pipeline. --- acme/acme/client_test.py | 4 +- acme/acme/crypto_util.py | 10 +--- acme/acme/standalone_test.py | 18 +++++--- acme/acme/test_util.py | 9 ++++ appveyor.yml | 30 ++++++++++-- certbot/compat.py | 12 ++++- certbot/crypto_util.py | 7 ++- certbot/display/completer.py | 4 +- certbot/display/util.py | 2 +- certbot/main.py | 3 +- certbot/plugins/manual_test.py | 16 ++++--- certbot/plugins/util_test.py | 7 +-- certbot/plugins/webroot_test.py | 12 ++--- certbot/renewal.py | 2 +- certbot/tests/account_test.py | 10 ++++ certbot/tests/cert_manager_test.py | 4 +- certbot/tests/cli_test.py | 1 + certbot/tests/crypto_util_test.py | 3 +- certbot/tests/display/completer_test.py | 13 ++++-- certbot/tests/display/util_test.py | 6 +-- certbot/tests/error_handler_test.py | 10 ++++ certbot/tests/hook_test.py | 1 + certbot/tests/lock_test.py | 2 + certbot/tests/log_test.py | 3 +- certbot/tests/main_test.py | 81 +++++++++++++++++++++------------ certbot/tests/reverter_test.py | 10 ++++ certbot/tests/storage_test.py | 1 + certbot/tests/util.py | 46 ++++++++++++++++++- certbot/tests/util_test.py | 28 ++++++------ certbot/util.py | 4 +- tox-win.ini | 13 ++++++ 31 files changed, 269 insertions(+), 103 deletions(-) create mode 100644 tox-win.ini diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 4f8a1abe2..acfd7eaff 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -1038,8 +1038,8 @@ class ClientNetworkTest(unittest.TestCase): # Requests Library Exceptions except requests.exceptions.ConnectionError as z: #pragma: no cover - self.assertEqual("('Connection aborted.', " - "error(111, 'Connection refused'))", str(z)) + self.assertTrue("('Connection aborted.', error(111, 'Connection refused'))" + == str(z) or "[WinError 10061]" in str(z)) class ClientNetworkWithMockedResponseTest(unittest.TestCase): """Tests for acme.client.ClientNetwork which mock out response.""" diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index d0e203417..c88cab943 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -136,22 +136,16 @@ def probe_sni(name, host, port=443, timeout=300, socket_kwargs = {'source_address': source_address} - host_protocol_agnostic = host - if host == '::' or host == '0': - # https://github.com/python/typeshed/pull/2136 - # while PR is not merged, we need to ignore - host_protocol_agnostic = None - try: # pylint: disable=star-args logger.debug( - "Attempting to connect to %s:%d%s.", host_protocol_agnostic, port, + "Attempting to connect to %s:%d%s.", host, port, " from {0}:{1}".format( source_address[0], source_address[1] ) if socket_kwargs else "" ) - socket_tuple = (host_protocol_agnostic, port) # type: Tuple[Optional[str], int] + socket_tuple = (host, port) # type: Tuple[str, int] sock = socket.create_connection(socket_tuple, **socket_kwargs) # type: ignore except socket.error as error: raise errors.Error(error) diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 6beea038e..ee527782a 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -48,7 +48,7 @@ class TLSSNI01ServerTest(unittest.TestCase): test_util.load_cert('rsa2048_cert.pem'), )} from acme.standalone import TLSSNI01Server - self.server = TLSSNI01Server(("", 0), certs=self.certs) + self.server = TLSSNI01Server(('localhost', 0), certs=self.certs) # pylint: disable=no-member self.thread = threading.Thread(target=self.server.serve_forever) self.thread.start() @@ -133,8 +133,11 @@ class BaseDualNetworkedServersTest(unittest.TestCase): self.address_family = socket.AF_INET socketserver.TCPServer.__init__(self, *args, **kwargs) if ipv6: + # NB: On Windows, socket.IPPROTO_IPV6 constant may be missing. + # We use the corresponding value (41) instead. + level = getattr(socket, "IPPROTO_IPV6", 41) # pylint: disable=no-member - self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1) + self.socket.setsockopt(level, socket.IPV6_V6ONLY, 1) try: self.server_bind() self.server_activate() @@ -147,15 +150,15 @@ class BaseDualNetworkedServersTest(unittest.TestCase): mock_bind.side_effect = socket.error from acme.standalone import BaseDualNetworkedServers self.assertRaises(socket.error, BaseDualNetworkedServers, - BaseDualNetworkedServersTest.SingleProtocolServer, - ("", 0), - socketserver.BaseRequestHandler) + BaseDualNetworkedServersTest.SingleProtocolServer, + ('', 0), + socketserver.BaseRequestHandler) def test_ports_equal(self): from acme.standalone import BaseDualNetworkedServers servers = BaseDualNetworkedServers( BaseDualNetworkedServersTest.SingleProtocolServer, - ("", 0), + ('', 0), socketserver.BaseRequestHandler) socknames = servers.getsocknames() prev_port = None @@ -177,7 +180,7 @@ class TLSSNI01DualNetworkedServersTest(unittest.TestCase): test_util.load_cert('rsa2048_cert.pem'), )} from acme.standalone import TLSSNI01DualNetworkedServers - self.servers = TLSSNI01DualNetworkedServers(("", 0), certs=self.certs) + self.servers = TLSSNI01DualNetworkedServers(('localhost', 0), certs=self.certs) self.servers.serve_forever() def tearDown(self): @@ -245,6 +248,7 @@ class HTTP01DualNetworkedServersTest(unittest.TestCase): self.assertFalse(self._test_http01(add=False)) +@test_util.broken_on_windows class TestSimpleTLSSNI01Server(unittest.TestCase): """Tests for acme.standalone.simple_tls_sni_01_server.""" diff --git a/acme/acme/test_util.py b/acme/acme/test_util.py index 1a0b67056..f97614700 100644 --- a/acme/acme/test_util.py +++ b/acme/acme/test_util.py @@ -4,6 +4,7 @@ """ import os +import sys import pkg_resources import unittest @@ -94,3 +95,11 @@ def skip_unless(condition, reason): # pragma: no cover return lambda cls: cls else: return lambda cls: None + +def broken_on_windows(function): + """Decorator to skip temporarily a broken test on Windows.""" + reason = 'Test is broken and ignored on windows but should be fixed.' + return unittest.skipIf( + sys.platform == 'win32' + and os.environ.get('SKIP_BROKEN_TESTS_ON_WINDOWS', 'true') == 'true', + reason)(function) diff --git a/appveyor.yml b/appveyor.yml index 67ad67c16..2d1c463ac 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,12 +1,34 @@ -# AppVeyor CI pipeline, executed on Windows Server 2016/2012 R2 +image: + # => Windows Server 2012 R2 + - Visual Studio 2015 + # => Windows Server 2016 + - Visual Studio 2017 + branches: only: - master - - /^\d+\.\d+\.x$/ # version branches like X.X.X + - /^\d+\.\d+\.x$/ # Version branches like X.X.X - /^test-.*$/ +install: + # Fail fast if a newer build is queued for the same PR + - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod ` + https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | ` + Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { ` + throw "There are newer queued builds for this pull request, failing early." } + # Use Python 3.7 by default + - "SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%" + # Check env + - "python --version" + # Upgrade pip to avoid warnings + - "python -m pip install --upgrade pip" + # Ready to install tox and coverage + - "pip install tox codecov" + build: off test_script: - - ps: Write-Host "Hello, world!" - \ No newline at end of file + - tox -c tox-win.ini -e py34,py35,py36,py37-cover + +on_success: + - codecov diff --git a/certbot/compat.py b/certbot/compat.py index 1dc89dfd8..e3e1bc4e1 100644 --- a/certbot/compat.py +++ b/certbot/compat.py @@ -2,7 +2,7 @@ Compatibility layer to run certbot both on Linux and Windows. The approach used here is similar to Modernizr for Web browsers. -We do not check the plateform type to determine if a particular logic is supported. +We do not check the platform type to determine if a particular logic is supported. Instead, we apply a logic, and then fallback to another logic if first logic is not supported at runtime. @@ -13,6 +13,7 @@ import select import sys import errno import ctypes +import stat from certbot import errors @@ -138,3 +139,12 @@ def release_locked_file(fd, path): raise finally: os.close(fd) + +def compare_file_modes(mode1, mode2): + """Return true if the two modes can be considered as equals for this platform""" + if 'fcntl' in sys.modules: + # Linux specific: standard compare + return oct(stat.S_IMODE(mode1)) == oct(stat.S_IMODE(mode2)) + # Windows specific: most of mode bits are ignored on Windows. Only check user R/W rights. + return (stat.S_IMODE(mode1) & stat.S_IREAD == stat.S_IMODE(mode2) & stat.S_IREAD + and stat.S_IMODE(mode1) & stat.S_IWRITE == stat.S_IMODE(mode2) & stat.S_IWRITE) diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 6193a8fbf..942f8502f 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -449,14 +449,17 @@ def _notAfterBefore(cert_path, method): def sha256sum(filename): """Compute a sha256sum of a file. + NB: In given file, platform specific newlines characters will be converted + into their equivalent unicode counterparts before calculating the hash. + :param str filename: path to the file whose hash will be computed :returns: sha256 digest of the file in hexadecimal :rtype: str """ sha256 = hashlib.sha256() - with open(filename, 'rb') as f: - sha256.update(f.read()) + with open(filename, 'rU') as file_d: + sha256.update(file_d.read().encode('UTF-8')) return sha256.hexdigest() def cert_and_chain_from_fullchain(fullchain_pem): diff --git a/certbot/display/completer.py b/certbot/display/completer.py index 08b55fdea..509a1051a 100644 --- a/certbot/display/completer.py +++ b/certbot/display/completer.py @@ -49,9 +49,9 @@ class Completer(object): readline.set_completer(self.complete) readline.set_completer_delims(' \t\n;') - # readline can be implemented using GNU readline or libedit + # readline can be implemented using GNU readline, pyreadline or libedit # which have different configuration syntax - if 'libedit' in readline.__doc__: + if readline.__doc__ is not None and 'libedit' in readline.__doc__: readline.parse_and_bind('bind ^I rl_complete') else: readline.parse_and_bind('tab: complete') diff --git a/certbot/display/util.py b/certbot/display/util.py index 9a813d4b7..772b67d74 100644 --- a/certbot/display/util.py +++ b/certbot/display/util.py @@ -53,7 +53,7 @@ def _wrap_lines(msg): break_long_words=False, break_on_hyphens=False)) - return os.linesep.join(fixed_l) + return '\n'.join(fixed_l) def input_with_timeout(prompt=None, timeout=36000.0): diff --git a/certbot/main.py b/certbot/main.py index 6cd2bbfac..5d5251dd2 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1135,7 +1135,8 @@ def _csr_get_and_save_cert(config, le_client): "Dry run: skipping saving certificate to %s", config.cert_path) return None, None cert_path, _, fullchain_path = le_client.save_certificate( - cert, chain, config.cert_path, config.chain_path, config.fullchain_path) + cert, chain, os.path.normpath(config.cert_path), + os.path.normpath(config.chain_path), os.path.normpath(config.fullchain_path)) return cert_path, fullchain_path def renew_cert(config, plugins, lineage): diff --git a/certbot/plugins/manual_test.py b/certbot/plugins/manual_test.py index ba4eb6889..0938e8a7d 100644 --- a/certbot/plugins/manual_test.py +++ b/certbot/plugins/manual_test.py @@ -4,6 +4,7 @@ import unittest import six import mock +import sys from acme import challenges @@ -75,12 +76,14 @@ class AuthenticatorTest(test_util.TempDirTestCase): def test_script_perform(self): self.config.manual_public_ip_logging_ok = True self.config.manual_auth_hook = ( - 'echo ${CERTBOT_DOMAIN}; ' - 'echo ${CERTBOT_TOKEN:-notoken}; ' - 'echo ${CERTBOT_CERT_PATH:-nocert}; ' - 'echo ${CERTBOT_KEY_PATH:-nokey}; ' - 'echo ${CERTBOT_SNI_DOMAIN:-nosnidomain}; ' - 'echo ${CERTBOT_VALIDATION:-novalidation};') + '{0} -c "from __future__ import print_function;' + 'import os; print(os.environ.get(\'CERTBOT_DOMAIN\'));' + 'print(os.environ.get(\'CERTBOT_TOKEN\', \'notoken\'));' + 'print(os.environ.get(\'CERTBOT_CERT_PATH\', \'nocert\'));' + 'print(os.environ.get(\'CERTBOT_KEY_PATH\', \'nokey\'));' + 'print(os.environ.get(\'CERTBOT_SNI_DOMAIN\', \'nosnidomain\'));' + 'print(os.environ.get(\'CERTBOT_VALIDATION\', \'novalidation\'));"' + .format(sys.executable)) dns_expected = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}'.format( self.dns_achall.domain, 'notoken', 'nocert', 'nokey', 'nosnidomain', @@ -128,6 +131,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): achall.validation(achall.account_key) in args[0]) self.assertFalse(kwargs['wrap']) + @test_util.broken_on_windows def test_cleanup(self): self.config.manual_public_ip_logging_ok = True self.config.manual_auth_hook = 'echo foo;' diff --git a/certbot/plugins/util_test.py b/certbot/plugins/util_test.py index b2f9c79ea..8ecd380b8 100644 --- a/certbot/plugins/util_test.py +++ b/certbot/plugins/util_test.py @@ -4,13 +4,14 @@ import unittest import mock - class GetPrefixTest(unittest.TestCase): """Tests for certbot.plugins.get_prefixes.""" def test_get_prefix(self): from certbot.plugins.util import get_prefixes - self.assertEqual(get_prefixes('/a/b/c'), ['/a/b/c', '/a/b', '/a', '/']) - self.assertEqual(get_prefixes('/'), ['/']) + self.assertEqual( + get_prefixes('/a/b/c'), + [os.path.normpath(path) for path in ['/a/b/c', '/a/b', '/a', '/']]) + self.assertEqual(get_prefixes('/'), [os.path.normpath('/')]) self.assertEqual(get_prefixes('a'), ['a']) class PathSurgeryTest(unittest.TestCase): diff --git a/certbot/plugins/webroot_test.py b/certbot/plugins/webroot_test.py index 59133f0aa..5303fe4da 100644 --- a/certbot/plugins/webroot_test.py +++ b/certbot/plugins/webroot_test.py @@ -4,9 +4,9 @@ from __future__ import print_function import argparse import errno +import json import os import shutil -import stat import tempfile import unittest @@ -17,6 +17,7 @@ import six from acme import challenges from certbot import achallenges +from certbot import compat from certbot import errors from certbot.display import util as display_util @@ -142,6 +143,7 @@ class AuthenticatorTest(unittest.TestCase): self.assertRaises(errors.PluginError, self.auth.perform, []) os.chmod(self.path, 0o700) + @test_util.skip_on_windows('On Windows, there is no chown.') @mock.patch("certbot.plugins.webroot.os.chown") def test_failed_chown(self, mock_chown): mock_chown.side_effect = OSError(errno.EACCES, "msg") @@ -169,16 +171,14 @@ class AuthenticatorTest(unittest.TestCase): # Remove exec bit from permission check, so that it # matches the file self.auth.perform([self.achall]) - path_permissions = stat.S_IMODE(os.stat(self.validation_path).st_mode) - self.assertEqual(path_permissions, 0o644) + self.assertTrue(compat.compare_file_modes(os.stat(self.validation_path).st_mode, 0o644)) # Check permissions of the directories for dirpath, dirnames, _ in os.walk(self.path): for directory in dirnames: full_path = os.path.join(dirpath, directory) - dir_permissions = stat.S_IMODE(os.stat(full_path).st_mode) - self.assertEqual(dir_permissions, 0o755) + self.assertTrue(compat.compare_file_modes(os.stat(full_path).st_mode, 0o755)) parent_gid = os.stat(self.path).st_gid parent_uid = os.stat(self.path).st_uid @@ -274,7 +274,7 @@ class WebrootActionTest(unittest.TestCase): def test_webroot_map_action(self): args = self.parser.parse_args( - ["--webroot-map", '{{"thing.com":"{0}"}}'.format(self.path)]) + ["--webroot-map", json.dumps({'thing.com': self.path})]) self.assertEqual(args.webroot_map["thing.com"], self.path) def test_domain_before_webroot(self): diff --git a/certbot/renewal.py b/certbot/renewal.py index ecc8b1f2f..a1508fa60 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -301,7 +301,7 @@ def renew_cert(config, domains, le_client, lineage): domains = lineage.names() # The private key is the existing lineage private key if reuse_key is set. # Otherwise, generate a fresh private key by passing None. - new_key = lineage.privkey if config.reuse_key else None + new_key = os.path.normpath(lineage.privkey) if config.reuse_key else None new_cert, new_chain, new_key, _ = le_client.obtain_certificate(domains, new_key) if config.dry_run: logger.debug("Dry run: skipping updating lineage at %s", diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index b2be47d0f..278b0c545 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -116,6 +116,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase): def test_init_creates_dir(self): self.assertTrue(os.path.isdir(self.config.accounts_dir)) + @test_util.broken_on_windows def test_save_and_restore(self): self.storage.save(self.acc, self.mock_client) account_path = os.path.join(self.config.accounts_dir, self.acc.id) @@ -218,12 +219,14 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self._set_server('https://acme-staging.api.letsencrypt.org/directory') self.assertEqual([], self.storage.find_all()) + @test_util.broken_on_windows def test_upgrade_version_staging(self): self._set_server('https://acme-staging.api.letsencrypt.org/directory') self.storage.save(self.acc, self.mock_client) self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') self.assertEqual([self.acc], self.storage.find_all()) + @test_util.broken_on_windows def test_upgrade_version_production(self): self._set_server('https://acme-v01.api.letsencrypt.org/directory') self.storage.save(self.acc, self.mock_client) @@ -241,6 +244,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') self.assertEqual([], self.storage.find_all()) + @test_util.broken_on_windows def test_upgrade_load(self): self._set_server('https://acme-staging.api.letsencrypt.org/directory') self.storage.save(self.acc, self.mock_client) @@ -249,6 +253,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase): account = self.storage.load(self.acc.id) self.assertEqual(prev_account, account) + @test_util.broken_on_windows def test_upgrade_load_single_account(self): self._set_server('https://acme-staging.api.letsencrypt.org/directory') self.storage.save(self.acc, self.mock_client) @@ -273,6 +278,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase): errors.AccountStorageError, self.storage.save, self.acc, self.mock_client) + @test_util.broken_on_windows def test_delete(self): self.storage.save(self.acc, self.mock_client) self.storage.delete(self.acc.id) @@ -307,10 +313,12 @@ class AccountFileStorageTest(test_util.ConfigTestCase): self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory') self.assertRaises(errors.AccountNotFound, self.storage.load, self.acc.id) + @test_util.broken_on_windows def test_delete_folders_up(self): self._test_delete_folders('https://acme-staging.api.letsencrypt.org/directory') self._assert_symlinked_account_removed() + @test_util.broken_on_windows def test_delete_folders_down(self): self._test_delete_folders('https://acme-staging-v02.api.letsencrypt.org/directory') self._assert_symlinked_account_removed() @@ -320,10 +328,12 @@ class AccountFileStorageTest(test_util.ConfigTestCase): with open(os.path.join(self.config.accounts_dir, 'foo'), 'w') as f: f.write('bar') + @test_util.broken_on_windows def test_delete_shared_account_up(self): self._set_server_and_stop_symlink('https://acme-staging-v02.api.letsencrypt.org/directory') self._test_delete_folders('https://acme-staging.api.letsencrypt.org/directory') + @test_util.broken_on_windows def test_delete_shared_account_down(self): self._set_server_and_stop_symlink('https://acme-staging-v02.api.letsencrypt.org/directory') self._test_delete_folders('https://acme-staging-v02.api.letsencrypt.org/directory') diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index 6ec1d4f5c..1aef33b0c 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -204,7 +204,7 @@ class CertificatesTest(BaseCertManagerTest): shutil.rmtree(empty_tempdir) @mock.patch('certbot.cert_manager.ocsp.RevocationChecker.ocsp_revoked') - def test_report_human_readable(self, mock_revoked): + def test_report_human_readable(self, mock_revoked): #pylint: disable=too-many-statements mock_revoked.return_value = None from certbot import cert_manager import datetime, pytz @@ -228,7 +228,7 @@ class CertificatesTest(BaseCertManagerTest): cert.target_expiry += datetime.timedelta(hours=2) # pylint: disable=protected-access out = get_report() - self.assertTrue('1 hour(s)' in out) + self.assertTrue('1 hour(s)' in out or '2 hour(s)' in out) self.assertTrue('VALID' in out and not 'INVALID' in out) cert.target_expiry += datetime.timedelta(days=1) diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 979cd97c1..69ef16597 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -76,6 +76,7 @@ class ParseTest(unittest.TestCase): # pylint: disable=too-many-public-methods return output.getvalue() + @test_util.broken_on_windows @mock.patch("certbot.cli.flag_default") def test_cli_ini_domains(self, mock_flag_default): tmp_config = tempfile.NamedTemporaryFile() diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index baf14b2ef..c092efc51 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -140,7 +140,7 @@ class ImportCSRFileTest(unittest.TestCase): util.CSR(file=csrfile, data=data_pem, form="pem"), - ["Example.com"],), + ["Example.com"]), self._call(csrfile, data)) def test_pem_csr(self): @@ -376,7 +376,6 @@ class NotAfterTest(unittest.TestCase): class Sha256sumTest(unittest.TestCase): """Tests for certbot.crypto_util.notAfter""" - def test_sha256sum(self): from certbot.crypto_util import sha256sum self.assertEqual(sha256sum(CERT_PATH), diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index ac01103b8..455bf5e1e 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -1,6 +1,9 @@ """Test certbot.display.completer.""" import os -import readline +try: + import readline # pylint: disable=import-error +except ImportError: + import certbot.display.dummy_readline as readline # type: ignore import string import sys import unittest @@ -9,9 +12,9 @@ import mock from six.moves import reload_module # pylint: disable=import-error from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module -from certbot.tests.util import TempDirTestCase +import certbot.tests.util as test_util -class CompleterTest(TempDirTestCase): +class CompleterTest(test_util.TempDirTestCase): """Test certbot.display.completer.Completer.""" def setUp(self): @@ -47,6 +50,8 @@ class CompleterTest(TempDirTestCase): completion = my_completer.complete(self.tempdir, num_paths) self.assertEqual(completion, None) + @unittest.skipIf('readline' not in sys.modules, + reason='Not relevant if readline is not available.') def test_import_error(self): original_readline = sys.modules['readline'] sys.modules['readline'] = None @@ -91,7 +96,7 @@ class CompleterTest(TempDirTestCase): def enable_tab_completion(unused_command): """Enables readline tab completion using the system specific syntax.""" - libedit = 'libedit' in readline.__doc__ + libedit = readline.__doc__ is not None and 'libedit' in readline.__doc__ command = 'bind ^I rl_complete' if libedit else 'tab: complete' readline.parse_and_bind(command) diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index f5cf29047..5672a20bd 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -1,6 +1,5 @@ """Test :mod:`certbot.display.util`.""" import inspect -import os import socket import tempfile import unittest @@ -10,7 +9,6 @@ import mock from certbot import errors from certbot import interfaces - from certbot.display import util as display_util @@ -281,10 +279,10 @@ class FileOutputDisplayTest(unittest.TestCase): msg = ("This is just a weak test{0}" "This function is only meant to be for easy viewing{0}" "Test a really really really really really really really really " - "really really really really long line...".format(os.linesep)) + "really really really really long line...".format('\n')) text = display_util._wrap_lines(msg) - self.assertEqual(text.count(os.linesep), 3) + self.assertEqual(text.count('\n'), 3) def test_get_valid_int_ans_valid(self): # pylint: disable=protected-access diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index a4a65e2d4..8508a3df5 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -10,6 +10,7 @@ import mock from acme.magic_typing import Callable, Dict, Union # pylint: enable=unused-import, no-name-in-module +import certbot.tests.util as test_util def get_signals(signums): """Get the handlers for an iterable of signums.""" @@ -65,6 +66,8 @@ class ErrorHandlerTest(unittest.TestCase): self.init_func.assert_called_once_with(*self.init_args, **self.init_kwargs) + # On Windows, this test kills pytest itself ! + @test_util.broken_on_windows def test_context_manager_with_signal(self): init_signals = get_signals(self.signals) with signal_receiver(self.signals) as signals_received: @@ -95,6 +98,8 @@ class ErrorHandlerTest(unittest.TestCase): **self.init_kwargs) bad_func.assert_called_once_with() + # On Windows, this test kills pytest itself ! + @test_util.broken_on_windows def test_bad_recovery_with_signal(self): sig1 = self.signals[0] sig2 = self.signals[-1] @@ -144,5 +149,10 @@ class ExitHandlerTest(ErrorHandlerTest): **self.init_kwargs) func.assert_called_once_with() + # On Windows, this test kills pytest itself ! + @test_util.broken_on_windows + def test_bad_recovery_with_signal(self): + super(ExitHandlerTest, self).test_bad_recovery_with_signal() + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index c9cfc69f9..f5bb0c8b5 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -37,6 +37,7 @@ class ValidateHookTest(util.TempDirTestCase): from certbot.hooks import validate_hook return validate_hook(*args, **kwargs) + @util.broken_on_windows def test_not_executable(self): file_path = os.path.join(self.tempdir, "foo") # create a non-executable file diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index 51469e8c1..aa82701f3 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -10,6 +10,7 @@ from certbot import errors from certbot.tests import util as test_util +@test_util.broken_on_windows class LockDirTest(test_util.TempDirTestCase): """Tests for certbot.lock.lock_dir.""" @classmethod @@ -24,6 +25,7 @@ class LockDirTest(test_util.TempDirTestCase): test_util.lock_and_call(assert_raises, lock_path) +@test_util.broken_on_windows class LockFileTest(test_util.TempDirTestCase): """Tests for certbot.lock.LockFile.""" @classmethod diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index c5991347e..6588bf5ca 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -12,6 +12,7 @@ import six from acme import messages from acme.magic_typing import Optional # pylint: disable=unused-import, no-name-in-module +from certbot import compat from certbot import constants from certbot import errors from certbot import util @@ -259,7 +260,7 @@ class TempHandlerTest(unittest.TestCase): def test_permissions(self): self.assertTrue( - util.check_permissions(self.handler.path, 0o600, os.getuid())) + util.check_permissions(self.handler.path, 0o600, compat.os_geteuid())) def test_delete(self): self.handler.close() diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 8334068c9..8d6a3e7ae 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -4,6 +4,7 @@ from __future__ import print_function import itertools +import json import mock import os import shutil @@ -11,6 +12,8 @@ import traceback import unittest import datetime import pytz +import tempfile +import sys import josepy as jose import six @@ -588,12 +591,14 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertTrue(message in str(exc)) self.assertTrue(exc is not None) + @test_util.broken_on_windows def test_noninteractive(self): args = ['-n', 'certonly'] self._cli_missing_flag(args, "specify a plugin") args.extend(['--standalone', '-d', 'eg.is']) self._cli_missing_flag(args, "register before running") + @test_util.broken_on_windows @mock.patch('certbot.main._report_new_cert') @mock.patch('certbot.main.client.acme_client.Client') @mock.patch('certbot.main._determine_account') @@ -635,43 +640,46 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met @mock.patch('certbot.main.plug_sel.record_chosen_plugins') @mock.patch('certbot.main.plug_sel.pick_installer') def test_installer_certname(self, _inst, _rec, mock_install): - mock_lineage = mock.MagicMock(cert_path="/tmp/cert", chain_path="/tmp/chain", - fullchain_path="/tmp/chain", - key_path="/tmp/privkey") + mock_lineage = mock.MagicMock(cert_path=test_util.temp_join('cert'), + chain_path=test_util.temp_join('chain'), + fullchain_path=test_util.temp_join('chain'), + key_path=test_util.temp_join('privkey')) with mock.patch("certbot.cert_manager.lineage_for_certname") as mock_getlin: mock_getlin.return_value = mock_lineage self._call(['install', '--cert-name', 'whatever'], mockisfile=True) call_config = mock_install.call_args[0][0] - self.assertEqual(call_config.cert_path, "/tmp/cert") - self.assertEqual(call_config.fullchain_path, "/tmp/chain") - self.assertEqual(call_config.key_path, "/tmp/privkey") + self.assertEqual(call_config.cert_path, test_util.temp_join('cert')) + self.assertEqual(call_config.fullchain_path, test_util.temp_join('chain')) + self.assertEqual(call_config.key_path, test_util.temp_join('privkey')) + @test_util.broken_on_windows @mock.patch('certbot.main._install_cert') @mock.patch('certbot.main.plug_sel.record_chosen_plugins') @mock.patch('certbot.main.plug_sel.pick_installer') def test_installer_param_override(self, _inst, _rec, mock_install): - mock_lineage = mock.MagicMock(cert_path="/tmp/cert", chain_path="/tmp/chain", - fullchain_path="/tmp/chain", - key_path="/tmp/privkey") + mock_lineage = mock.MagicMock(cert_path=test_util.temp_join('cert'), + chain_path=test_util.temp_join('chain'), + fullchain_path=test_util.temp_join('chain'), + key_path=test_util.temp_join('privkey')) with mock.patch("certbot.cert_manager.lineage_for_certname") as mock_getlin: mock_getlin.return_value = mock_lineage self._call(['install', '--cert-name', 'whatever', - '--key-path', '/tmp/overriding_privkey'], mockisfile=True) + '--key-path', test_util.temp_join('overriding_privkey')], mockisfile=True) call_config = mock_install.call_args[0][0] - self.assertEqual(call_config.cert_path, "/tmp/cert") - self.assertEqual(call_config.fullchain_path, "/tmp/chain") - self.assertEqual(call_config.chain_path, "/tmp/chain") - self.assertEqual(call_config.key_path, "/tmp/overriding_privkey") + self.assertEqual(call_config.cert_path, test_util.temp_join('cert')) + self.assertEqual(call_config.fullchain_path, test_util.temp_join('chain')) + self.assertEqual(call_config.chain_path, test_util.temp_join('chain')) + self.assertEqual(call_config.key_path, test_util.temp_join('overriding_privkey')) mock_install.reset() self._call(['install', '--cert-name', 'whatever', - '--cert-path', '/tmp/overriding_cert'], mockisfile=True) + '--cert-path', test_util.temp_join('overriding_cert')], mockisfile=True) call_config = mock_install.call_args[0][0] - self.assertEqual(call_config.cert_path, "/tmp/overriding_cert") - self.assertEqual(call_config.fullchain_path, "/tmp/chain") - self.assertEqual(call_config.key_path, "/tmp/privkey") + self.assertEqual(call_config.cert_path, test_util.temp_join('overriding_cert')) + self.assertEqual(call_config.fullchain_path, test_util.temp_join('chain')) + self.assertEqual(call_config.key_path, test_util.temp_join('privkey')) @mock.patch('certbot.main.plug_sel.record_chosen_plugins') @mock.patch('certbot.main.plug_sel.pick_installer') @@ -686,15 +694,17 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met @mock.patch('certbot.cert_manager.get_certnames') @mock.patch('certbot.main._install_cert') def test_installer_select_cert(self, mock_inst, mock_getcert, _inst, _rec): - mock_lineage = mock.MagicMock(cert_path="/tmp/cert", chain_path="/tmp/chain", - fullchain_path="/tmp/chain", - key_path="/tmp/privkey") + mock_lineage = mock.MagicMock(cert_path=test_util.temp_join('cert'), + chain_path=test_util.temp_join('chain'), + fullchain_path=test_util.temp_join('chain'), + key_path=test_util.temp_join('privkey')) with mock.patch("certbot.cert_manager.lineage_for_certname") as mock_getlin: mock_getlin.return_value = mock_lineage self._call(['install'], mockisfile=True) self.assertTrue(mock_getcert.called) self.assertTrue(mock_inst.called) + @test_util.broken_on_windows @mock.patch('certbot.main._report_new_cert') @mock.patch('certbot.util.exe_exists') def test_configurator_selection(self, mock_exe_exists, unused_report): @@ -710,7 +720,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met # ret, _, _, _ = self._call(args) # self.assertTrue("Too many flags setting" in ret) - args = ["install", "--nginx", "--cert-path", "/tmp/blah", "--key-path", "/tmp/blah", + args = ["install", "--nginx", "--cert-path", + test_util.temp_join('blah'), "--key-path", test_util.temp_join('blah'), "--nginx-server-root", "/nonexistent/thing", "-d", "example.com", "--debug"] if "nginx" in real_plugins: @@ -733,6 +744,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self._call(["auth", "--standalone"]) self.assertEqual(1, mock_certonly.call_count) + @test_util.broken_on_windows def test_rollback(self): _, _, _, client = self._call(['rollback']) self.assertEqual(1, client.rollback.call_count) @@ -760,6 +772,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self._call_no_clientmock(['delete']) self.assertEqual(1, mock_cert_manager.call_count) + @test_util.broken_on_windows def test_plugins(self): flags = ['--init', '--prepare', '--authenticators', '--installers'] for args in itertools.chain( @@ -1014,7 +1027,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met # The location of the previous live privkey.pem is passed # to obtain_certificate mock_client.obtain_certificate.assert_called_once_with(['isnot.org'], - os.path.join(self.config.config_dir, "live/sample-renewal/privkey.pem")) + os.path.normpath(os.path.join( + self.config.config_dir, "live/sample-renewal/privkey.pem"))) else: mock_client.obtain_certificate.assert_called_once_with(['isnot.org'], None) else: @@ -1039,6 +1053,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertTrue('fullchain.pem' in cert_msg) self.assertTrue('donate' in get_utility().add_message.call_args[0][0]) + @test_util.broken_on_windows @mock.patch('certbot.crypto_util.notAfter') def test_certonly_renewal_triggers(self, unused_notafter): # --dry-run should force renewal @@ -1087,6 +1102,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertTrue('No renewals were attempted.' in stdout.getvalue()) self.assertTrue('The following certs are not due for renewal yet:' in stdout.getvalue()) + @test_util.broken_on_windows def test_quiet_renew(self): test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf') args = ["renew", "--dry-run"] @@ -1204,7 +1220,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met renewalparams = {'authenticator': 'webroot'} self._test_renew_common( renewalparams=renewalparams, assert_oc_called=True, - args=['renew', '--webroot-map', '{"example.com": "/tmp"}']) + args=['renew', '--webroot-map', json.dumps({'example.com': tempfile.gettempdir()})]) def test_renew_reconstitute_error(self): # pylint: disable=protected-access @@ -1234,7 +1250,9 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met def test_no_renewal_with_hooks(self): _, _, stdout = self._test_renewal_common( due_for_renewal=False, extra_args=None, should_renew=False, - args=['renew', '--post-hook', 'echo hello world']) + args=['renew', '--post-hook', + '{0} -c "from __future__ import print_function; print(\'hello world\');"' + .format(sys.executable)]) self.assertTrue('No hooks were run.' in stdout.getvalue()) @test_util.patch_get_utility() @@ -1254,13 +1272,19 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met chain = 'chain' mock_client = mock.MagicMock() mock_client.obtain_certificate_from_csr.return_value = (certr, chain) - cert_path = '/etc/letsencrypt/live/example.com/cert_512.pem' - full_path = '/etc/letsencrypt/live/example.com/fullchain.pem' + cert_path = os.path.normpath(os.path.join( + self.config.config_dir, + 'live/example.com/cert_512.pem')) + full_path = os.path.normpath(os.path.join( + self.config.config_dir, + 'live/example.com/fullchain.pem')) mock_client.save_certificate.return_value = cert_path, None, full_path with mock.patch('certbot.main._init_le_client') as mock_init: mock_init.return_value = mock_client with test_util.patch_get_utility() as mock_get_utility: - chain_path = '/etc/letsencrypt/live/example.com/chain.pem' + chain_path = os.path.normpath(os.path.join( + self.config.config_dir, + 'live/example.com/chain.pem')) args = ('-a standalone certonly --csr {0} --cert-path {1} ' '--chain-path {2} --fullchain-path {3}').format( CSR, cert_path, chain_path, full_path).split() @@ -1334,6 +1358,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self._call(['-c', test_util.vector_path('cli.ini')]) self.assertTrue(mocked_run.called) + @test_util.broken_on_windows def test_register(self): with mock.patch('certbot.main.client') as mocked_client: acc = mock.MagicMock() diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index b048737c2..999c6225c 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -50,6 +50,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): x = f.read() self.assertTrue("No changes" in x) + @test_util.broken_on_windows def test_basic_add_to_temp_checkpoint(self): # These shouldn't conflict even though they are both named config.txt self.reverter.add_to_temp_checkpoint(self.sets[0], "save1") @@ -91,6 +92,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): self.assertRaises(errors.ReverterError, self.reverter.add_to_checkpoint, set([config3]), "invalid save") + @test_util.broken_on_windows def test_multiple_saves_and_temp_revert(self): self.reverter.add_to_temp_checkpoint(self.sets[0], "save1") update_file(self.config1, "updated-directive") @@ -120,6 +122,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): self.assertFalse(os.path.isfile(config3)) self.assertFalse(os.path.isfile(config4)) + @test_util.broken_on_windows def test_multiple_registration_same_file(self): self.reverter.register_file_creation(True, self.config1) self.reverter.register_file_creation(True, self.config1) @@ -144,6 +147,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): errors.ReverterError, self.reverter.register_file_creation, "filepath") + @test_util.broken_on_windows def test_register_undo_command(self): coms = [ ["a2dismod", "ssl"], @@ -166,6 +170,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): errors.ReverterError, self.reverter.register_undo_command, True, ["command"]) + @test_util.broken_on_windows @mock.patch("certbot.util.run_script") def test_run_undo_commands(self, mock_run): mock_run.side_effect = ["", errors.SubprocessError] @@ -229,6 +234,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): self.assertRaises( errors.ReverterError, self.reverter.revert_temporary_config) + @test_util.broken_on_windows @mock.patch("certbot.reverter.logger.warning") def test_recover_checkpoint_missing_new_files(self, mock_warn): self.reverter.register_file_creation( @@ -243,6 +249,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): self.assertRaises( errors.ReverterError, self.reverter.revert_temporary_config) + @test_util.broken_on_windows def test_recovery_routine_temp_and_perm(self): # Register a new perm checkpoint file config3 = os.path.join(self.dir1, "config3.txt") @@ -306,6 +313,7 @@ class TestFullCheckpointsReverter(test_util.ConfigTestCase): self.assertRaises( errors.ReverterError, self.reverter.rollback_checkpoints, "one") + @test_util.broken_on_windows def test_rollback_finalize_checkpoint_valid_inputs(self): config3 = self._setup_three_checkpoints() @@ -357,6 +365,7 @@ class TestFullCheckpointsReverter(test_util.ConfigTestCase): self.assertRaises( errors.ReverterError, self.reverter.finalize_checkpoint, "Title") + @test_util.broken_on_windows @mock.patch("certbot.reverter.logger") def test_rollback_too_many(self, mock_logger): # Test no exist warning... @@ -369,6 +378,7 @@ class TestFullCheckpointsReverter(test_util.ConfigTestCase): self.reverter.rollback_checkpoints(4) self.assertEqual(mock_logger.warning.call_count, 1) + @test_util.broken_on_windows def test_multi_rollback(self): config3 = self._setup_three_checkpoints() self.reverter.rollback_checkpoints(3) diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 078a2858f..03d595652 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -480,6 +480,7 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertTrue(self.test_rc.should_autorenew()) mock_ocsp.return_value = False + @test_util.broken_on_windows @mock.patch("certbot.storage.relevant_values") def test_save_successor(self, mock_rv): # Mock relevant_values() to claim that all values are relevant here diff --git a/certbot/tests/util.py b/certbot/tests/util.py index d505ea76c..822597dd4 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -9,6 +9,8 @@ import pkg_resources import shutil import tempfile import unittest +import sys +import warnings from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization @@ -36,8 +38,15 @@ def vector_path(*names): def load_vector(*names): """Load contents of a test vector.""" # luckily, resource_string opens file in binary mode - return pkg_resources.resource_string( + data = pkg_resources.resource_string( __name__, os.path.join('testdata', *names)) + # Try at most to convert CRLF to LF when data is text + try: + return data.decode().replace('\r\n', '\n').encode() + except ValueError: + # Failed to process the file with standard encoding. + # Most likely not a text file, return its bytes untouched. + return data def _guess_loader(filename, loader_pem, loader_der): @@ -314,10 +323,21 @@ class TempDirTestCase(unittest.TestCase): """Base test class which sets up and tears down a temporary directory""" def setUp(self): + """Execute before test""" self.tempdir = tempfile.mkdtemp() def tearDown(self): - shutil.rmtree(self.tempdir) + """Execute after test""" + # Then we have various files which are not correctly closed at the time of tearDown. + # On Windows, it is visible for the same reasons as above. + # For know, we log them until a proper file close handling is written. + def onerror_handler(_, path, excinfo): + """On error handler""" + message = ('Following error occurred when deleting the tempdir {0}' + ' for path {1} during tearDown process: {2}' + .format(self.tempdir, path, str(excinfo))) + warnings.warn(message) + shutil.rmtree(self.tempdir, onerror=onerror_handler) class ConfigTestCase(TempDirTestCase): """Test class which sets up a NamespaceConfig object. @@ -378,3 +398,25 @@ def hold_lock(cv, lock_path): # pragma: no cover cv.notify() cv.wait() my_lock.release() + +def skip_on_windows(reason): + """Decorator to skip permanently a test on Windows. A reason is required.""" + def wrapper(function): + """Wrapped version""" + return unittest.skipIf(sys.platform == 'win32', reason)(function) + return wrapper + +def broken_on_windows(function): + """Decorator to skip temporarily a broken test on Windows.""" + reason = 'Test is broken and ignored on windows but should be fixed.' + return unittest.skipIf( + sys.platform == 'win32' + and os.environ.get('SKIP_BROKEN_TESTS_ON_WINDOWS', 'true') == 'true', + reason)(function) + +def temp_join(path): + """ + Return the given path joined to the tempdir path for the current platform + Eg.: 'cert' => /tmp/cert (Linux) or 'C:\\Users\\currentuser\\AppData\\Temp\\cert' (Windows) + """ + return os.path.join(tempfile.gettempdir(), path) diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 689b4108d..45cc55249 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -3,7 +3,6 @@ import argparse import errno import os import shutil -import stat import unittest import mock @@ -89,6 +88,7 @@ class LockDirUntilExit(test_util.TempDirTestCase): import certbot.util reload_module(certbot.util) + @test_util.broken_on_windows @mock.patch('certbot.util.logger') @mock.patch('certbot.util.atexit_register') def test_it(self, mock_register, mock_logger): @@ -140,9 +140,9 @@ class MakeOrVerifyDirTest(test_util.TempDirTestCase): super(MakeOrVerifyDirTest, self).setUp() self.path = os.path.join(self.tempdir, "foo") - os.mkdir(self.path, 0o400) + os.mkdir(self.path, 0o600) - self.uid = os.getuid() + self.uid = compat.os_geteuid() def _call(self, directory, mode): from certbot.util import make_or_verify_dir @@ -152,14 +152,15 @@ class MakeOrVerifyDirTest(test_util.TempDirTestCase): path = os.path.join(self.tempdir, "bar") self._call(path, 0o650) self.assertTrue(os.path.isdir(path)) - self.assertEqual(stat.S_IMODE(os.stat(path).st_mode), 0o650) + self.assertTrue(compat.compare_file_modes(os.stat(path).st_mode, 0o650)) def test_existing_correct_mode_does_not_fail(self): - self._call(self.path, 0o400) - self.assertEqual(stat.S_IMODE(os.stat(self.path).st_mode), 0o400) + self._call(self.path, 0o600) + self.assertTrue(compat.compare_file_modes(os.stat(self.path).st_mode, 0o600)) + @test_util.skip_on_windows('Umask modes are mostly ignored on Windows.') def test_existing_wrong_mode_fails(self): - self.assertRaises(errors.Error, self._call, self.path, 0o600) + self.assertRaises(errors.Error, self._call, self.path, 0o400) def test_reraises_os_error(self): with mock.patch.object(os, "makedirs") as makedirs: @@ -178,7 +179,7 @@ class CheckPermissionsTest(test_util.TempDirTestCase): def setUp(self): super(CheckPermissionsTest, self).setUp() - self.uid = os.getuid() + self.uid = compat.os_geteuid() def _call(self, mode): from certbot.util import check_permissions @@ -212,8 +213,8 @@ class UniqueFileTest(test_util.TempDirTestCase): self.assertEqual(open(name).read(), "bar") def test_right_mode(self): - self.assertEqual(0o700, os.stat(self._call(0o700)[1]).st_mode & 0o777) - self.assertEqual(0o100, os.stat(self._call(0o100)[1]).st_mode & 0o777) + self.assertTrue(compat.compare_file_modes(0o700, os.stat(self._call(0o700)[1]).st_mode)) + self.assertTrue(compat.compare_file_modes(0o600, os.stat(self._call(0o600)[1]).st_mode)) def test_default_exists(self): name1 = self._call()[1] # create 0000_foo.txt @@ -513,17 +514,16 @@ class OsInfoTest(unittest.TestCase): def test_systemd_os_release(self): from certbot.util import (get_os_info, get_systemd_os_info, - get_os_info_ua) + get_os_info_ua) with mock.patch('os.path.isfile', return_value=True): self.assertEqual(get_os_info( test_util.vector_path("os-release"))[0], 'systemdos') self.assertEqual(get_os_info( test_util.vector_path("os-release"))[1], '42') - self.assertEqual(get_systemd_os_info("/dev/null"), ("", "")) + self.assertEqual(get_systemd_os_info(os.devnull), ("", "")) self.assertEqual(get_os_info_ua( - test_util.vector_path("os-release")), - "SystemdOS") + test_util.vector_path("os-release")), "SystemdOS") with mock.patch('os.path.isfile', return_value=False): self.assertEqual(get_systemd_os_info(), ("", "")) diff --git a/certbot/util.py b/certbot/util.py index 8e84c29ba..d7c542465 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -12,7 +12,6 @@ import platform import re import six import socket -import stat import subprocess import sys @@ -21,6 +20,7 @@ from collections import OrderedDict import configargparse from acme.magic_typing import Tuple, Union # pylint: disable=unused-import, no-name-in-module +from certbot import compat from certbot import constants from certbot import errors from certbot import lock @@ -204,7 +204,7 @@ def check_permissions(filepath, mode, uid=0): """ file_stat = os.stat(filepath) - return stat.S_IMODE(file_stat.st_mode) == mode and file_stat.st_uid == uid + return compat.compare_file_modes(file_stat.st_mode, mode) and file_stat.st_uid == uid def safe_open(path, mode="w", chmod=None, buffering=None): diff --git a/tox-win.ini b/tox-win.ini new file mode 100644 index 000000000..fe063c264 --- /dev/null +++ b/tox-win.ini @@ -0,0 +1,13 @@ +[tox] +skipsdist = True +envlist = py{34,35,36,37}-cover + +[testenv] +deps = -e acme[dev] + -e .[dev] +commands = pytest -n auto --pyargs acme + pytest -n auto --pyargs certbot + +[testenv:cover] +commands = pytest -n auto --cov acme --pyargs acme + pytest -n auto --cov certbot --cov-append --pyargs certbot -- cgit v1.2.3 From 7b17c84dd97eeab474ceb96c1093fc735381ee33 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Sat, 20 Oct 2018 02:37:22 +0200 Subject: Remove custom code for fail fast because rolling builds in AppVeyor are enabled. (#6431) --- appveyor.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 2d1c463ac..796070081 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,11 +11,6 @@ branches: - /^test-.*$/ install: - # Fail fast if a newer build is queued for the same PR - - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod ` - https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | ` - Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { ` - throw "There are newer queued builds for this pull request, failing early." } # Use Python 3.7 by default - "SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%" # Check env -- cgit v1.2.3 From 36ebce4a5fa30a0e73bd8656aea54ded2e9d982e Mon Sep 17 00:00:00 2001 From: ohemorange Date: Fri, 19 Oct 2018 19:16:54 -0700 Subject: Fix ranking of vhosts in Nginx so that all port-matching vhosts come first (#6412) To more closely match how Nginx ranks things. --- CHANGELOG.md | 1 + certbot-nginx/certbot_nginx/configurator.py | 44 +++++++++---- .../certbot_nginx/tests/configurator_test.py | 75 +++++++++++++--------- 3 files changed, 79 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5060a7038..0f44258af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Fixed * Match Nginx parser update in allowing variable names to start with `${`. +* Fix ranking of vhosts in Nginx so that all port-matching vhosts come first * Correct OVH integration tests on machines without internet access. * Stop caching the results of ipv6_info in http01.py diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index d526381a2..0f9c43e2d 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -8,7 +8,6 @@ import tempfile import time import OpenSSL -import six import zope.interface from acme import challenges @@ -32,6 +31,12 @@ from certbot_nginx import obj # pylint: disable=unused-import from acme.magic_typing import List, Dict, Set # pylint: disable=unused-import, no-name-in-module +NAME_RANK = 0 +START_WILDCARD_RANK = 1 +END_WILDCARD_RANK = 2 +REGEX_RANK = 3 +NO_SSL_MODIFIER = 4 + logger = logging.getLogger(__name__) @@ -405,7 +410,8 @@ class NginxConfigurator(common.Installer): """ if not matches: return None - elif matches[0]['rank'] in six.moves.range(2, 6): + elif matches[0]['rank'] in [START_WILDCARD_RANK, END_WILDCARD_RANK, + START_WILDCARD_RANK + NO_SSL_MODIFIER, END_WILDCARD_RANK + NO_SSL_MODIFIER]: # Wildcard match - need to find the longest one rank = matches[0]['rank'] wildcards = [x for x in matches if x['rank'] == rank] @@ -414,10 +420,9 @@ class NginxConfigurator(common.Installer): # Exact or regex match return matches[0]['vhost'] - - def _rank_matches_by_name_and_ssl(self, vhost_list, target_name): + def _rank_matches_by_name(self, vhost_list, target_name): """Returns a ranked list of vhosts from vhost_list that match target_name. - The ranking gives preference to SSL vhosts. + This method should always be followed by a call to _select_best_name_match. :param list vhost_list: list of vhosts to filter and rank :param str target_name: The name to match @@ -437,21 +442,37 @@ class NginxConfigurator(common.Installer): if name_type == 'exact': matches.append({'vhost': vhost, 'name': name, - 'rank': 0 if vhost.ssl else 1}) + 'rank': NAME_RANK}) elif name_type == 'wildcard_start': matches.append({'vhost': vhost, 'name': name, - 'rank': 2 if vhost.ssl else 3}) + 'rank': START_WILDCARD_RANK}) elif name_type == 'wildcard_end': matches.append({'vhost': vhost, 'name': name, - 'rank': 4 if vhost.ssl else 5}) + 'rank': END_WILDCARD_RANK}) elif name_type == 'regex': matches.append({'vhost': vhost, 'name': name, - 'rank': 6 if vhost.ssl else 7}) + 'rank': REGEX_RANK}) return sorted(matches, key=lambda x: x['rank']) + def _rank_matches_by_name_and_ssl(self, vhost_list, target_name): + """Returns a ranked list of vhosts from vhost_list that match target_name. + The ranking gives preference to SSLishness before name match level. + + :param list vhost_list: list of vhosts to filter and rank + :param str target_name: The name to match + :returns: list of dicts containing the vhost, the matching name, and + the numerical rank + :rtype: list + + """ + matches = self._rank_matches_by_name(vhost_list, target_name) + for match in matches: + if not match['vhost'].ssl: + match['rank'] += NO_SSL_MODIFIER + return sorted(matches, key=lambda x: x['rank']) def choose_redirect_vhosts(self, target_name, port, create_if_no_match=False): """Chooses a single virtual host for redirect enhancement. @@ -531,9 +552,7 @@ class NginxConfigurator(common.Installer): matching_vhosts = [vhost for vhost in all_vhosts if _vhost_matches(vhost, port)] - # We can use this ranking function because sslishness doesn't matter to us, and - # there shouldn't be conflicting plaintextish servers listening on 80. - return self._rank_matches_by_name_and_ssl(matching_vhosts, target_name) + return self._rank_matches_by_name(matching_vhosts, target_name) def get_all_names(self): """Returns all names found in the Nginx Configuration. @@ -568,6 +587,7 @@ class NginxConfigurator(common.Installer): return util.get_filtered_names(all_names) def _get_snakeoil_paths(self): + """Generate invalid certs that let us create ssl directives for Nginx""" # TODO: generate only once tmp_dir = os.path.join(self.config.work_dir, "snakeoil") le_key = crypto_util.init_save_key( diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 4d23f3518..b180bb930 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -128,22 +128,39 @@ class NginxConfiguratorTest(util.NginxTest): ['#', parser.COMMENT]]]], parsed[0]) - def test_choose_vhosts(self): - localhost_conf = set(['localhost', r'~^(www\.)?(example|bar)\.']) - server_conf = set(['somename', 'another.alias', 'alias']) - example_conf = set(['.example.com', 'example.*']) - foo_conf = set(['*.www.foo.com', '*.www.example.com']) - ipv6_conf = set(['ipv6.com']) - - results = {'localhost': localhost_conf, - 'alias': server_conf, - 'example.com': example_conf, - 'example.com.uk.test': example_conf, - 'www.example.com': example_conf, - 'test.www.example.com': foo_conf, - 'abc.www.foo.com': foo_conf, - 'www.bar.co.uk': localhost_conf, - 'ipv6.com': ipv6_conf} + def test_choose_vhosts_alias(self): + self._test_choose_vhosts_common('alias', 'server_conf') + + def test_choose_vhosts_example_com(self): + self._test_choose_vhosts_common('example.com', 'example_conf') + + def test_choose_vhosts_localhost(self): + self._test_choose_vhosts_common('localhost', 'localhost_conf') + + def test_choose_vhosts_example_com_uk_test(self): + self._test_choose_vhosts_common('example.com.uk.test', 'example_conf') + + def test_choose_vhosts_www_example_com(self): + self._test_choose_vhosts_common('www.example.com', 'example_conf') + + def test_choose_vhosts_test_www_example_com(self): + self._test_choose_vhosts_common('test.www.example.com', 'foo_conf') + + def test_choose_vhosts_abc_www_foo_com(self): + self._test_choose_vhosts_common('abc.www.foo.com', 'foo_conf') + + def test_choose_vhosts_www_bar_co_uk(self): + self._test_choose_vhosts_common('www.bar.co.uk', 'localhost_conf') + + def test_choose_vhosts_ipv6_com(self): + self._test_choose_vhosts_common('ipv6.com', 'ipv6_conf') + + def _test_choose_vhosts_common(self, name, conf): + conf_names = {'localhost_conf': set(['localhost', r'~^(www\.)?(example|bar)\.']), + 'server_conf': set(['somename', 'another.alias', 'alias']), + 'example_conf': set(['.example.com', 'example.*']), + 'foo_conf': set(['*.www.foo.com', '*.www.example.com']), + 'ipv6_conf': set(['ipv6.com'])} conf_path = {'localhost': "etc_nginx/nginx.conf", 'alias': "etc_nginx/nginx.conf", @@ -155,22 +172,22 @@ class NginxConfiguratorTest(util.NginxTest): 'www.bar.co.uk': "etc_nginx/nginx.conf", 'ipv6.com': "etc_nginx/sites-enabled/ipv6.com"} + vhost = self.config.choose_vhosts(name)[0] + path = os.path.relpath(vhost.filep, self.temp_dir) + + self.assertEqual(conf_names[conf], vhost.names) + self.assertEqual(conf_path[name], path) + # IPv6 specific checks + if name == "ipv6.com": + self.assertTrue(vhost.ipv6_enabled()) + # Make sure that we have SSL enabled also for IPv6 addr + self.assertTrue( + any([True for x in vhost.addrs if x.ssl and x.ipv6])) + + def test_choose_vhosts_bad(self): bad_results = ['www.foo.com', 'example', 't.www.bar.co', '69.255.225.155'] - for name in results: - vhost = self.config.choose_vhosts(name)[0] - path = os.path.relpath(vhost.filep, self.temp_dir) - - self.assertEqual(results[name], vhost.names) - self.assertEqual(conf_path[name], path) - # IPv6 specific checks - if name == "ipv6.com": - self.assertTrue(vhost.ipv6_enabled()) - # Make sure that we have SSL enabled also for IPv6 addr - self.assertTrue( - any([True for x in vhost.addrs if x.ssl and x.ipv6])) - for name in bad_results: self.assertRaises(errors.MisconfigurationError, self.config.choose_vhosts, name) -- cgit v1.2.3 From 9264561944fc847e4a5d2de9a2be1c7da811349d Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Mon, 29 Oct 2018 23:56:30 +0100 Subject: Check pattern for both old and new openssl (#6450) --- tests/certbot-boulder-integration.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/certbot-boulder-integration.sh b/tests/certbot-boulder-integration.sh index e250e591b..73e668e28 100755 --- a/tests/certbot-boulder-integration.sh +++ b/tests/certbot-boulder-integration.sh @@ -394,7 +394,7 @@ openssl x509 -in "${root}/csr/cert-p384.pem" -text | grep 'ASN1 OID: secp384r1' # OCSP Must Staple common auth --must-staple --domains "must-staple.le.wtf" -openssl x509 -in "${root}/conf/live/must-staple.le.wtf/cert.pem" -text | grep '1.3.6.1.5.5.7.1.24' +openssl x509 -in "${root}/conf/live/must-staple.le.wtf/cert.pem" -text | grep -E 'status_request|1\.3\.6\.1\.5\.5\.7\.1\.24' # revoke by account key common revoke --cert-path "$root/conf/live/le.wtf/cert.pem" --delete-after-revoke -- cgit v1.2.3 From 1d783fd4b9c83c669b33adb2afaff8ec21e00cb6 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 31 Oct 2018 18:34:14 +0200 Subject: Update Augeas lens to fix some Apache configuration parsing issues (#6438) * Update Augeas lens to fix some Apache configuration parsing issues * Added CHANGELOG entry --- CHANGELOG.md | 1 + .../certbot_apache/augeas_lens/httpd.aug | 112 +++++++++++++++++---- 2 files changed, 91 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f44258af..1e0ba82ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Fix ranking of vhosts in Nginx so that all port-matching vhosts come first * Correct OVH integration tests on machines without internet access. * Stop caching the results of ipv6_info in http01.py +* The grammar used by Augeas parser in Apache plugin was updated to fix various parsing errors. ## 0.27.1 - 2018-09-06 diff --git a/certbot-apache/certbot_apache/augeas_lens/httpd.aug b/certbot-apache/certbot_apache/augeas_lens/httpd.aug index 2729f4b60..5600088cf 100644 --- a/certbot-apache/certbot_apache/augeas_lens/httpd.aug +++ b/certbot-apache/certbot_apache/augeas_lens/httpd.aug @@ -44,67 +44,134 @@ autoload xfm *****************************************************************) let dels (s:string) = del s s +(* The continuation sequence that indicates that we should consider the + * next line part of the current line *) +let cont = /\\\\\r?\n/ + +(* Whitespace within a line: space, tab, and the continuation sequence *) +let ws = /[ \t]/ | cont + +(* Any possible character - '.' does not match \n *) +let any = /(.|\n)/ + +(* Any character preceded by a backslash *) +let esc_any = /\\\\(.|\n)/ + +(* Newline sequence - both for Unix and DOS newlines *) +let nl = /\r?\n/ + +(* Whitespace at the end of a line *) +let eol = del (ws* . nl) "\n" + (* deal with continuation lines *) -let sep_spc = del /([ \t]+|[ \t]*\\\\\r?\n[ \t]*)+/ " " -let sep_osp = del /([ \t]*|[ \t]*\\\\\r?\n[ \t]*)*/ "" -let sep_eq = del /[ \t]*=[ \t]*/ "=" +let sep_spc = del ws+ " " +let sep_osp = del ws* "" +let sep_eq = del (ws* . "=" . ws*) "=" let nmtoken = /[a-zA-Z:_][a-zA-Z0-9:_.-]*/ let word = /[a-z][a-z0-9._-]*/i -let eol = Util.doseol -let empty = Util.empty_dos +(* A complete line that is either just whitespace or a comment that only + * contains whitespace *) +let empty = [ del (ws* . /#?/ . ws* . nl) "\n" ] + let indent = Util.indent -let comment_val_re = /([^ \t\r\n](.|\\\\\r?\n)*[^ \\\t\r\n]|[^ \t\r\n])/ -let comment = [ label "#comment" . del /[ \t]*#[ \t]*/ "# " - . store comment_val_re . eol ] +(* A comment that is not just whitespace. We define it in terms of the + * things that are not allowed as part of such a comment: + * 1) Starts with whitespace + * 2) Ends with whitespace, a backslash or \r + * 3) Unescaped newlines + *) +let comment = + let comment_start = del (ws* . "#" . ws* ) "# " in + let unesc_eol = /[^\]?/ . nl in + let w = /[^\t\n\r \\]/ in + let r = /[\r\\]/ in + let s = /[\t\r ]/ in + (* + * we'd like to write + * let b = /\\\\/ in + * let t = /[\t\n\r ]/ in + * let x = b . (t? . (s|w)* ) in + * but the definition of b depends on commit 244c0edd in 1.9.0 and + * would make the lens unusable with versions before 1.9.0. So we write + * x out which works in older versions, too + *) + let x = /\\\\[\t\n\r ]?[^\n\\]*/ in + let line = ((r . s* . w|w|r) . (s|w)* . x*|(r.s* )?).w.(s*.w)* in + [ label "#comment" . comment_start . store line . eol ] (* borrowed from shellvars.aug *) -let char_arg_dir = /([^\\ '"{\t\r\n]|[^ '"{\t\r\n]+[^\\ \t\r\n])|\\\\"|\\\\'|\\\\ / let char_arg_sec = /([^\\ '"\t\r\n>]|[^ '"\t\r\n>]+[^\\ \t\r\n>])|\\\\"|\\\\'|\\\\ / let char_arg_wl = /([^\\ '"},\t\r\n]|[^ '"},\t\r\n]+[^\\ '"},\t\r\n])/ -let cdot = /\\\\./ -let cl = /\\\\\n/ let dquot = let no_dquot = /[^"\\\r\n]/ - in /"/ . (no_dquot|cdot|cl)* . /"/ + in /"/ . (no_dquot|esc_any)* . /"/ let dquot_msg = let no_dquot = /([^ \t"\\\r\n]|[^"\\\r\n]+[^ \t"\\\r\n])/ - in /"/ . (no_dquot|cdot|cl)* + in /"/ . (no_dquot|esc_any)* . no_dquot + let squot = let no_squot = /[^'\\\r\n]/ - in /'/ . (no_squot|cdot|cl)* . /'/ + in /'/ . (no_squot|esc_any)* . /'/ let comp = /[<>=]?=/ (****************************************************************** * Attributes *****************************************************************) -let arg_dir = [ label "arg" . store (char_arg_dir+|dquot|squot) ] +(* The arguments for a directive come in two flavors: quoted with single or + * double quotes, or bare. Bare arguments may not start with a single or + * double quote; since we also treat "word lists" special, i.e. lists + * enclosed in curly braces, bare arguments may not start with those, + * either. + * + * Bare arguments may not contain unescaped spaces, but we allow escaping + * with '\\'. Quoted arguments can contain anything, though the quote must + * be escaped with '\\'. + *) +let bare = /([^{"' \t\n\r]|\\\\.)([^ \t\n\r]|\\\\.)*[^ \t\n\r\\]|[^{"' \t\n\r\\]/ + +let arg_quoted = [ label "arg" . store (dquot|squot) ] +let arg_bare = [ label "arg" . store bare ] + (* message argument starts with " but ends at EOL *) let arg_dir_msg = [ label "arg" . store dquot_msg ] -let arg_sec = [ label "arg" . store (char_arg_sec+|comp|dquot|squot) ] let arg_wl = [ label "arg" . store (char_arg_wl+|dquot|squot) ] (* comma-separated wordlist as permitted in the SSLRequire directive *) let arg_wordlist = - let wl_start = Util.del_str "{" in - let wl_end = Util.del_str "}" in + let wl_start = dels "{" in + let wl_end = dels "}" in let wl_sep = del /[ \t]*,[ \t]*/ ", " in [ label "wordlist" . wl_start . arg_wl . (wl_sep . arg_wl)* . wl_end ] let argv (l:lens) = l . (sep_spc . l)* +(* the arguments of a directive. We use this once we have parsed the name + * of the directive, and the space right after it. When dir_args is used, + * we also know that we have at least one argument. We need to be careful + * with the spacing between arguments: quoted arguments and word lists do + * not need to have space between them, but bare arguments do. + * + * Apache apparently is also happy if the last argument starts with a double + * quote, but has no corresponding closing duoble quote, which is what + * arg_dir_msg handles + *) +let dir_args = + let arg_nospc = arg_quoted|arg_wordlist in + (arg_bare . sep_spc | arg_nospc . sep_osp)* . (arg_bare|arg_nospc|arg_dir_msg) + let directive = - (* arg_dir_msg may be the last or only argument *) - let dir_args = (argv (arg_dir|arg_wordlist) . (sep_spc . arg_dir_msg)?) | arg_dir_msg - in [ indent . label "directive" . store word . (sep_spc . dir_args)? . eol ] + [ indent . label "directive" . store word . (sep_spc . dir_args)? . eol ] + +let arg_sec = [ label "arg" . store (char_arg_sec+|comp|dquot|squot) ] let section (body:lens) = (* opt_eol includes empty lines *) - let opt_eol = del /([ \t]*#?\r?\n)*/ "\n" in + let opt_eol = del /([ \t]*#?[ \t]*\r?\n)*/ "\n" in let inner = (sep_spc . argv arg_sec)? . sep_osp . dels ">" . opt_eol . ((body|comment) . (body|empty|comment)*)? . indent . dels " Date: Fri, 2 Nov 2018 18:59:27 +0200 Subject: Dummy AWS credentials for Route53 tests to prevent outbound connections (#6456) Boto3 / botocore library has a feature that tries to fetch AWS credentials from IAM if a set of credentials isn't available otherwise. This happens when boto loops through different credential providers in order to find the keys. See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=912103 This PR simply adds dummy environmental variables for the tests that will be picked up by the credential provider iterator in order to prevent making outbound connections. * Hardcode dummy AWS credentials to prevent boto3 making outgoing connections * Remove the dummy credentials when tearing down test case --- CHANGELOG.md | 1 + .../certbot_dns_route53/dns_route53_test.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e0ba82ad..f033b3691 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Fix ranking of vhosts in Nginx so that all port-matching vhosts come first * Correct OVH integration tests on machines without internet access. * Stop caching the results of ipv6_info in http01.py +* Test fix for Route53 plugin to prevent boto3 making outgoing connections. * The grammar used by Augeas parser in Apache plugin was updated to fix various parsing errors. ## 0.27.1 - 2018-09-06 diff --git a/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py b/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py index 7534e132c..71326c2af 100644 --- a/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py +++ b/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py @@ -1,5 +1,6 @@ """Tests for certbot_dns_route53.dns_route53.Authenticator""" +import os import unittest import mock @@ -20,8 +21,18 @@ class AuthenticatorTest(unittest.TestCase, dns_test_common.BaseAuthenticatorTest self.config = mock.MagicMock() + # Set up dummy credentials for testing + os.environ["AWS_ACCESS_KEY_ID"] = "dummy_access_key" + os.environ["AWS_SECRET_ACCESS_KEY"] = "dummy_secret_access_key" + self.auth = Authenticator(self.config, "route53") + def tearDown(self): + # Remove the dummy credentials from env vars + del os.environ["AWS_ACCESS_KEY_ID"] + del os.environ["AWS_SECRET_ACCESS_KEY"] + super(AuthenticatorTest, self).tearDown() + def test_perform(self): self.auth._change_txt_record = mock.MagicMock() self.auth._wait_for_change = mock.MagicMock() @@ -117,8 +128,18 @@ class ClientTest(unittest.TestCase): self.config = mock.MagicMock() + # Set up dummy credentials for testing + os.environ["AWS_ACCESS_KEY_ID"] = "dummy_access_key" + os.environ["AWS_SECRET_ACCESS_KEY"] = "dummy_secret_access_key" + self.client = Authenticator(self.config, "route53") + def tearDown(self): + # Remove the dummy credentials from env vars + del os.environ["AWS_ACCESS_KEY_ID"] + del os.environ["AWS_SECRET_ACCESS_KEY"] + super(ClientTest, self).tearDown() + def test_find_zone_id_for_domain(self): self.client.r53.get_paginator = mock.MagicMock() self.client.r53.get_paginator().paginate.return_value = [ -- cgit v1.2.3 From 2c1964c639916bcedc4d10d073cbcca212e7b8f1 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Fri, 2 Nov 2018 17:32:33 -0700 Subject: Use the ACMEv2 newNonce endpoint when a new nonce is needed (#6442) Also, add checking to the newNonce HEAD request, and check responses in general before attempting to save a nonce, for a better error message. * check response before adding nonce to the pool * fix tests so that they test what they're supposed to test, and also allow the order of _add_nonce and _check_response to be switched * make _get_nonce take acme_version * Send HEAD to newNonce endpoint when using ACMEv2 * check the HEAD newNonce response * remove unnecessary try; get returns None if the item doesn't exist * instead of setting new_nonce_url on ClientNetwork, use the saved directory in ClientBase and pass that into ClientNetwork.post * no need to test acme_version in _get_nonce * pop new_nonce_url out of kwargs before passing to _send_request --- CHANGELOG.md | 1 + acme/acme/client.py | 20 ++++++++++--- acme/acme/client_test.py | 74 +++++++++++++++++++++++++++++++++++++----------- 3 files changed, 75 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f033b3691..b33c4ad97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Added * `revoke` accepts `--cert-name`, and doesn't accept both `--cert-name` and `--cert-path`. +* Use the ACMEv2 newNonce endpoint when a new nonce is needed, and newNonce is available in the directory. ### Changed diff --git a/acme/acme/client.py b/acme/acme/client.py index bd86657b9..adc8ad9e3 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -89,6 +89,8 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes """ kwargs.setdefault('acme_version', self.acme_version) + if hasattr(self.directory, 'newNonce'): + kwargs.setdefault('new_nonce_url', getattr(self.directory, 'newNonce')) return self.net.post(*args, **kwargs) def update_registration(self, regr, update=None): @@ -1106,10 +1108,15 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes else: raise errors.MissingNonce(response) - def _get_nonce(self, url): + def _get_nonce(self, url, new_nonce_url): if not self._nonces: logger.debug('Requesting fresh nonce') - self._add_nonce(self.head(url)) + if new_nonce_url is None: + response = self.head(url) + else: + # request a new nonce from the acme newNonce endpoint + response = self._check_response(self.head(new_nonce_url), content_type=None) + self._add_nonce(response) return self._nonces.pop() def post(self, *args, **kwargs): @@ -1130,8 +1137,13 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes def _post_once(self, url, obj, content_type=JOSE_CONTENT_TYPE, acme_version=1, **kwargs): - data = self._wrap_in_jws(obj, self._get_nonce(url), url, acme_version) + try: + new_nonce_url = kwargs.pop('new_nonce_url') + except KeyError: + new_nonce_url = None + data = self._wrap_in_jws(obj, self._get_nonce(url, new_nonce_url), url, acme_version) kwargs.setdefault('headers', {'Content-Type': content_type}) response = self._send_request('POST', url, data=data, **kwargs) + response = self._check_response(response, content_type=content_type) self._add_nonce(response) - return self._check_response(response, content_type=content_type) + return response diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index acfd7eaff..2046d2377 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -805,7 +805,8 @@ class ClientV2Test(ClientTestBase): def test_revoke(self): self.client.revoke(messages_test.CERT, self.rsn) self.net.post.assert_called_once_with( - self.directory["revokeCert"], mock.ANY, acme_version=2) + self.directory["revokeCert"], mock.ANY, acme_version=2, + new_nonce_url=DIRECTORY_V2['newNonce']) def test_update_registration(self): # "Instance of 'Field' has no to_json/update member" bug: @@ -1052,7 +1053,10 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase): self.response = mock.MagicMock(ok=True, status_code=http_client.OK) self.response.headers = {} self.response.links = {} - self.checked_response = mock.MagicMock() + self.response.checked = False + self.acmev1_nonce_response = mock.MagicMock(ok=False, + status_code=http_client.METHOD_NOT_ALLOWED) + self.acmev1_nonce_response.headers = {} self.obj = mock.MagicMock() self.wrapped_obj = mock.MagicMock() self.content_type = mock.sentinel.content_type @@ -1064,13 +1068,21 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase): def send_request(*args, **kwargs): # pylint: disable=unused-argument,missing-docstring + self.assertFalse("new_nonce_url" in kwargs) + method = args[0] + uri = args[1] + if method == 'HEAD' and uri != "new_nonce_uri": + response = self.acmev1_nonce_response + else: + response = self.response + if self.available_nonces: - self.response.headers = { + response.headers = { self.net.REPLAY_NONCE_HEADER: self.available_nonces.pop().decode()} else: - self.response.headers = {} - return self.response + response.headers = {} + return response # pylint: disable=protected-access self.net._send_request = self.send_request = mock.MagicMock( @@ -1082,28 +1094,39 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase): # pylint: disable=missing-docstring self.assertEqual(self.response, response) self.assertEqual(self.content_type, content_type) - return self.checked_response + self.assertTrue(self.response.ok) + self.response.checked = True + return self.response def test_head(self): - self.assertEqual(self.response, self.net.head( + self.assertEqual(self.acmev1_nonce_response, self.net.head( 'http://example.com/', 'foo', bar='baz')) self.send_request.assert_called_once_with( 'HEAD', 'http://example.com/', 'foo', bar='baz') + def test_head_v2(self): + self.assertEqual(self.response, self.net.head( + 'new_nonce_uri', 'foo', bar='baz')) + self.send_request.assert_called_once_with( + 'HEAD', 'new_nonce_uri', 'foo', bar='baz') + def test_get(self): - self.assertEqual(self.checked_response, self.net.get( + self.assertEqual(self.response, self.net.get( 'http://example.com/', content_type=self.content_type, bar='baz')) + self.assertTrue(self.response.checked) self.send_request.assert_called_once_with( 'GET', 'http://example.com/', bar='baz') def test_post_no_content_type(self): self.content_type = self.net.JOSE_CONTENT_TYPE - self.assertEqual(self.checked_response, self.net.post('uri', self.obj)) + self.assertEqual(self.response, self.net.post('uri', self.obj)) + self.assertTrue(self.response.checked) def test_post(self): # pylint: disable=protected-access - self.assertEqual(self.checked_response, self.net.post( + self.assertEqual(self.response, self.net.post( 'uri', self.obj, content_type=self.content_type)) + self.assertTrue(self.response.checked) self.net._wrap_in_jws.assert_called_once_with( self.obj, jose.b64decode(self.all_nonces.pop()), "uri", 1) @@ -1135,7 +1158,7 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase): def test_post_not_retried(self): check_response = mock.MagicMock() check_response.side_effect = [messages.Error.with_code('malformed'), - self.checked_response] + self.response] # pylint: disable=protected-access self.net._check_response = check_response @@ -1143,13 +1166,12 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase): self.obj, content_type=self.content_type) def test_post_successful_retry(self): - check_response = mock.MagicMock() - check_response.side_effect = [messages.Error.with_code('badNonce'), - self.checked_response] + post_once = mock.MagicMock() + post_once.side_effect = [messages.Error.with_code('badNonce'), + self.response] # pylint: disable=protected-access - self.net._check_response = check_response - self.assertEqual(self.checked_response, self.net.post( + self.assertEqual(self.response, self.net.post( 'uri', self.obj, content_type=self.content_type)) def test_head_get_post_error_passthrough(self): @@ -1160,6 +1182,26 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase): self.assertRaises(requests.exceptions.RequestException, self.net.post, 'uri', obj=self.obj) + def test_post_bad_nonce_head(self): + # pylint: disable=protected-access + # regression test for https://github.com/certbot/certbot/issues/6092 + bad_response = mock.MagicMock(ok=False, status_code=http_client.SERVICE_UNAVAILABLE) + self.net._send_request = mock.MagicMock() + self.net._send_request.return_value = bad_response + self.content_type = None + check_response = mock.MagicMock() + self.net._check_response = check_response + self.assertRaises(errors.ClientError, self.net.post, 'uri', + self.obj, content_type=self.content_type, acme_version=2, + new_nonce_url='new_nonce_uri') + self.assertEqual(check_response.call_count, 1) + + def test_new_nonce_uri_removed(self): + self.content_type = None + self.net.post('uri', self.obj, content_type=None, + acme_version=2, new_nonce_url='new_nonce_uri') + + class ClientNetworkSourceAddressBindingTest(unittest.TestCase): """Tests that if ClientNetwork has a source IP set manually, the underlying library has used the provided source address.""" -- cgit v1.2.3 From 9403c1641dbe8a0755e0e18d890f50cbff99db37 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 5 Nov 2018 13:58:56 -0800 Subject: Stop preferring TLS-SNI in the Apache, Nginx, and standalone plugins (#6461) * flip challenge preference in Nginx * Fix Nginx tests * Flip challenge preference in Apache * Flip challenge preference in standalone * update changelog * continue to run with tls-sni in integration tests for coverage --- CHANGELOG.md | 1 + certbot-apache/certbot_apache/configurator.py | 2 +- certbot-nginx/certbot_nginx/configurator.py | 2 +- certbot-nginx/certbot_nginx/tests/configurator_test.py | 2 +- certbot-nginx/tests/boulder-integration.sh | 2 ++ certbot/plugins/standalone.py | 2 +- 6 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b33c4ad97..53855cac0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Removed documentation mentions of `#letsencrypt` IRC on Freenode. * Write README to the base of (config-dir)/live directory * `--manual` will explicitly warn users that earlier challenges should remain in place when setting up subsequent challenges. +* Stop preferring TLS-SNI in the Apache, Nginx, and standalone plugins ### Fixed diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index da632dc81..f431b9dab 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -2253,7 +2253,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): ########################################################################### def get_chall_pref(self, unused_domain): # pylint: disable=no-self-use """Return list of challenge preferences.""" - return [challenges.TLSSNI01, challenges.HTTP01] + return [challenges.HTTP01, challenges.TLSSNI01] def perform(self, achalls): """Perform the configuration related challenge. diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 0f9c43e2d..dd0bf9e8b 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -1039,7 +1039,7 @@ class NginxConfigurator(common.Installer): ########################################################################### def get_chall_pref(self, unused_domain): # pylint: disable=no-self-use """Return list of challenge preferences.""" - return [challenges.TLSSNI01, challenges.HTTP01] + return [challenges.HTTP01, challenges.TLSSNI01] # Entry point in main.py for performing challenges def perform(self, achalls): diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index b180bb930..2814cbb8c 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -103,7 +103,7 @@ class NginxConfiguratorTest(util.NginxTest): errors.PluginError, self.config.enhance, 'myhost', 'unknown_enhancement') def test_get_chall_pref(self): - self.assertEqual([challenges.TLSSNI01, challenges.HTTP01], + self.assertEqual([challenges.HTTP01, challenges.TLSSNI01], self.config.get_chall_pref('myhost')) def test_save(self): diff --git a/certbot-nginx/tests/boulder-integration.sh b/certbot-nginx/tests/boulder-integration.sh index 194413f1d..2a24e645f 100755 --- a/certbot-nginx/tests/boulder-integration.sh +++ b/certbot-nginx/tests/boulder-integration.sh @@ -39,6 +39,8 @@ nginx -v reload_nginx certbot_test_nginx --domains nginx.wtf run test_deployment_and_rollback nginx.wtf +certbot_test_nginx --domains nginx.wtf run --preferred-challenges tls-sni +test_deployment_and_rollback nginx.wtf certbot_test_nginx --domains nginx2.wtf --preferred-challenges http test_deployment_and_rollback nginx2.wtf # Overlapping location block and server-block-level return 301 diff --git a/certbot/plugins/standalone.py b/certbot/plugins/standalone.py index cb2e69511..16f872a3f 100644 --- a/certbot/plugins/standalone.py +++ b/certbot/plugins/standalone.py @@ -114,7 +114,7 @@ class ServerManager(object): return self._instances.copy() -SUPPORTED_CHALLENGES = [challenges.TLSSNI01, challenges.HTTP01] \ +SUPPORTED_CHALLENGES = [challenges.HTTP01, challenges.TLSSNI01] \ # type: List[Type[challenges.KeyAuthorizationChallenge]] -- cgit v1.2.3 From bc7763dd0fd86bfac0774a80f6357035200025b7 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Mon, 5 Nov 2018 23:07:09 +0100 Subject: Lexicon v3 compatibility (#6474) * Propagate correctly domain to lexicon providers * Pass required parameter to ovh provider * Fix all other lexicon-based dns plugins --- certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py | 1 + certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py | 1 + certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy.py | 1 + certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py | 1 + certbot-dns-linode/certbot_dns_linode/dns_linode.py | 1 + certbot-dns-luadns/certbot_dns_luadns/dns_luadns.py | 1 + certbot-dns-nsone/certbot_dns_nsone/dns_nsone.py | 1 + certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py | 1 + certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py | 1 + certbot/plugins/dns_common_lexicon.py | 7 ++++++- 10 files changed, 15 insertions(+), 1 deletion(-) diff --git a/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py b/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py index 674194fee..658db6072 100644 --- a/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py +++ b/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py @@ -70,6 +70,7 @@ class _CloudXNSLexiconClient(dns_common_lexicon.LexiconClient): super(_CloudXNSLexiconClient, self).__init__() self.provider = cloudxns.Provider({ + 'provider_name': 'cloudxns', 'auth_username': api_key, 'auth_token': secret_key, 'ttl': ttl, diff --git a/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py b/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py index f3a98567e..3eb56e37c 100644 --- a/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py +++ b/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py @@ -66,6 +66,7 @@ class _DNSimpleLexiconClient(dns_common_lexicon.LexiconClient): super(_DNSimpleLexiconClient, self).__init__() self.provider = dnsimple.Provider({ + 'provider_name': 'dnssimple', 'auth_token': token, 'ttl': ttl, }) diff --git a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy.py b/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy.py index 982edfdd3..4236ce37a 100644 --- a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy.py +++ b/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy.py @@ -72,6 +72,7 @@ class _DNSMadeEasyLexiconClient(dns_common_lexicon.LexiconClient): super(_DNSMadeEasyLexiconClient, self).__init__() self.provider = dnsmadeeasy.Provider({ + 'provider_name': 'dnsmadeeasy', 'auth_username': api_key, 'auth_token': secret_key, 'ttl': ttl, diff --git a/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py b/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py index 50bfce1ae..9c35e72ab 100644 --- a/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py +++ b/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py @@ -73,6 +73,7 @@ class _GehirnLexiconClient(dns_common_lexicon.LexiconClient): super(_GehirnLexiconClient, self).__init__() self.provider = gehirn.Provider({ + 'provider_name': 'gehirn', 'auth_token': api_token, 'auth_secret': api_secret, 'ttl': ttl, diff --git a/certbot-dns-linode/certbot_dns_linode/dns_linode.py b/certbot-dns-linode/certbot_dns_linode/dns_linode.py index cc29ce842..01da2cf60 100644 --- a/certbot-dns-linode/certbot_dns_linode/dns_linode.py +++ b/certbot-dns-linode/certbot_dns_linode/dns_linode.py @@ -62,6 +62,7 @@ class _LinodeLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, api_key): super(_LinodeLexiconClient, self).__init__() self.provider = linode.Provider({ + 'provider_name': 'linode', 'auth_token': api_key }) diff --git a/certbot-dns-luadns/certbot_dns_luadns/dns_luadns.py b/certbot-dns-luadns/certbot_dns_luadns/dns_luadns.py index 00b62e6e1..bd6a16f69 100644 --- a/certbot-dns-luadns/certbot_dns_luadns/dns_luadns.py +++ b/certbot-dns-luadns/certbot_dns_luadns/dns_luadns.py @@ -69,6 +69,7 @@ class _LuaDNSLexiconClient(dns_common_lexicon.LexiconClient): super(_LuaDNSLexiconClient, self).__init__() self.provider = luadns.Provider({ + 'provider_name': 'luadns', 'auth_username': email, 'auth_token': token, 'ttl': ttl, diff --git a/certbot-dns-nsone/certbot_dns_nsone/dns_nsone.py b/certbot-dns-nsone/certbot_dns_nsone/dns_nsone.py index 28db126c1..5f33efbba 100644 --- a/certbot-dns-nsone/certbot_dns_nsone/dns_nsone.py +++ b/certbot-dns-nsone/certbot_dns_nsone/dns_nsone.py @@ -66,6 +66,7 @@ class _NS1LexiconClient(dns_common_lexicon.LexiconClient): super(_NS1LexiconClient, self).__init__() self.provider = nsone.Provider({ + 'provider_name': 'nsone', 'auth_token': api_key, 'ttl': ttl, }) diff --git a/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py b/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py index c4ded7748..578ee8e89 100644 --- a/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py +++ b/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py @@ -78,6 +78,7 @@ class _OVHLexiconClient(dns_common_lexicon.LexiconClient): super(_OVHLexiconClient, self).__init__() self.provider = ovh.Provider({ + 'provider_name': 'ovh', 'auth_entrypoint': endpoint, 'auth_application_key': application_key, 'auth_application_secret': application_secret, diff --git a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py index 6f1c74b68..b892330f5 100644 --- a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py +++ b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py @@ -76,6 +76,7 @@ class _SakuraCloudLexiconClient(dns_common_lexicon.LexiconClient): super(_SakuraCloudLexiconClient, self).__init__() self.provider = sakuracloud.Provider({ + 'provider_name': 'sakuracloud', 'auth_token': api_token, 'auth_secret': api_secret, 'ttl': ttl, diff --git a/certbot/plugins/dns_common_lexicon.py b/certbot/plugins/dns_common_lexicon.py index 7a97fc950..f9610b816 100644 --- a/certbot/plugins/dns_common_lexicon.py +++ b/certbot/plugins/dns_common_lexicon.py @@ -68,7 +68,12 @@ class LexiconClient(object): for domain_name in domain_name_guesses: try: - self.provider.options['domain'] = domain_name + if hasattr(self.provider, 'options'): + # For Lexicon 2.x + self.provider.options['domain'] = domain_name + else: + # For Lexicon 3.x + self.provider.domain = domain_name self.provider.authenticate() -- cgit v1.2.3 From cb8dd8a428edb98c6d9750950e5572f5337de7d5 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 5 Nov 2018 14:50:20 -0800 Subject: Warn when using deprecated acme.challenges.TLSSNI01 (#6469) * Warn when using deprecated acme.challenges.TLSSNI01 * Update changelog * remove specific date from warning * add a raw assert for mypy optional type checking --- CHANGELOG.md | 1 + acme/acme/challenges.py | 6 ++++++ acme/acme/challenges_test.py | 22 ++++++++++++++++------ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53855cac0..eea0ebb23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Removed documentation mentions of `#letsencrypt` IRC on Freenode. * Write README to the base of (config-dir)/live directory * `--manual` will explicitly warn users that earlier challenges should remain in place when setting up subsequent challenges. +* Warn when using deprecated acme.challenges.TLSSNI01 * Stop preferring TLS-SNI in the Apache, Nginx, and standalone plugins ### Fixed diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 2f0bf004d..a65768228 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -4,6 +4,7 @@ import functools import hashlib import logging import socket +import warnings from cryptography.hazmat.primitives import hashes # type: ignore import josepy as jose @@ -493,6 +494,11 @@ class TLSSNI01(KeyAuthorizationChallenge): # boulder#962, ietf-wg-acme#22 #n = jose.Field("n", encoder=int, decoder=int) + def __init__(self, *args, **kwargs): + warnings.warn("TLS-SNI-01 is deprecated, and will stop working soon.", + DeprecationWarning, stacklevel=2) + super(TLSSNI01, self).__init__(*args, **kwargs) + def validation(self, account_key, **kwargs): """Generate validation. diff --git a/acme/acme/challenges_test.py b/acme/acme/challenges_test.py index 661d25a35..9307eb95b 100644 --- a/acme/acme/challenges_test.py +++ b/acme/acme/challenges_test.py @@ -1,5 +1,6 @@ """Tests for acme.challenges.""" import unittest +import warnings import josepy as jose import mock @@ -360,20 +361,29 @@ class TLSSNI01ResponseTest(unittest.TestCase): class TLSSNI01Test(unittest.TestCase): def setUp(self): - from acme.challenges import TLSSNI01 - self.msg = TLSSNI01( - token=jose.b64decode('a82d5ff8ef740d12881f6d3c2277ab2e')) self.jmsg = { 'type': 'tls-sni-01', 'token': 'a82d5ff8ef740d12881f6d3c2277ab2e', } + def _msg(self): + from acme.challenges import TLSSNI01 + with warnings.catch_warnings(record=True) as warn: + warnings.simplefilter("always") + msg = TLSSNI01( + token=jose.b64decode('a82d5ff8ef740d12881f6d3c2277ab2e')) + assert warn is not None # using a raw assert for mypy + self.assertTrue(len(warn) == 1) + self.assertTrue(issubclass(warn[-1].category, DeprecationWarning)) + self.assertTrue('deprecated' in str(warn[-1].message)) + return msg + def test_to_partial_json(self): - self.assertEqual(self.jmsg, self.msg.to_partial_json()) + self.assertEqual(self.jmsg, self._msg().to_partial_json()) def test_from_json(self): from acme.challenges import TLSSNI01 - self.assertEqual(self.msg, TLSSNI01.from_json(self.jmsg)) + self.assertEqual(self._msg(), TLSSNI01.from_json(self.jmsg)) def test_from_json_hashable(self): from acme.challenges import TLSSNI01 @@ -388,7 +398,7 @@ class TLSSNI01Test(unittest.TestCase): @mock.patch('acme.challenges.TLSSNI01Response.gen_cert') def test_validation(self, mock_gen_cert): mock_gen_cert.return_value = ('cert', 'key') - self.assertEqual(('cert', 'key'), self.msg.validation( + self.assertEqual(('cert', 'key'), self._msg().validation( KEY, cert_key=mock.sentinel.cert_key)) mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key) -- cgit v1.2.3 From cbdc2ee23b1020e7450c646688ca6b930af4c795 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 5 Nov 2018 15:01:16 -0800 Subject: Log warning about TLS-SNI deprecation in Certbot (#6468) For #6319. * print warning in auth_handler * add test --- certbot/auth_handler.py | 6 ++++++ certbot/tests/auth_handler_test.py | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index e7d658b25..efee49143 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -113,6 +113,12 @@ class AuthHandler(object): aauthzr.authzr, path) aauthzr.achalls.extend(aauthzr_achalls) + for aauthzr in aauthzrs: + for achall in aauthzr.achalls: + if isinstance(achall.chall, challenges.TLSSNI01): + logger.warning("TLS-SNI-01 is deprecated, and will stop working soon.") + return + def _has_challenges(self, aauthzrs): """Do we have any challenges to perform?""" return any(aauthzr.achalls for aauthzr in aauthzrs) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 76d1df90f..e1319b614 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -327,6 +327,11 @@ class HandleAuthorizationsTest(unittest.TestCase): azr.body.combinations) aauthzrs[i] = type(aauthzr)(updated_azr, aauthzr.achalls) + @mock.patch("certbot.auth_handler.logger") + def test_tls_sni_logs(self, logger): + self._test_name1_tls_sni_01_1_common(combos=True) + self.assertTrue("deprecated" in logger.warning.call_args[0][0]) + class PollChallengesTest(unittest.TestCase): # pylint: disable=protected-access -- cgit v1.2.3 From 47062dbfbfa4aaedc5910461e3f1bf14686ce27b Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 5 Nov 2018 17:09:03 -0800 Subject: update changelog (#6476) --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eea0ebb23..df95c2d36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Write README to the base of (config-dir)/live directory * `--manual` will explicitly warn users that earlier challenges should remain in place when setting up subsequent challenges. * Warn when using deprecated acme.challenges.TLSSNI01 +* Log warning about TLS-SNI deprecation in Certbot * Stop preferring TLS-SNI in the Apache, Nginx, and standalone plugins ### Fixed -- cgit v1.2.3 From 4edfb3ef65ee2522f5b09c88793095ebee886bab Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 7 Nov 2018 00:35:09 +0100 Subject: [Windows] Handle file renaming when the destination path already exists (#6415) On Linux, you can invoke os.rename(src, dst) even if dst already exists. In this case, destination file will be atomically replaced by the source file. On Windows, this will lead to an OSError because changes are not atomic. This cause certbot renew to fail in particular, because the old certificate configuration needs to be replace by the new when a certificate is effectively renewed. One could use the cross-platform function os.replace, but it is available only on Python >= 3.3. This PR add a function in compat to handle correctly this case on Windows, and delegating everything else to os.rename. * Cross platform compatible os.rename (we can use os.replace if its python 3) * Use os.replace instead of custom non-atomic code. * Avoid errors for lint and mypy. Add a test. --- certbot/compat.py | 24 ++++++++++++++++++++++++ certbot/reverter.py | 2 +- certbot/storage.py | 3 ++- certbot/tests/compat_test.py | 21 +++++++++++++++++++++ certbot/tests/reverter_test.py | 2 +- 5 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 certbot/tests/compat_test.py diff --git a/certbot/compat.py b/certbot/compat.py index e3e1bc4e1..d42febe81 100644 --- a/certbot/compat.py +++ b/certbot/compat.py @@ -65,6 +65,30 @@ def os_geteuid(): # Windows specific return 0 +def os_rename(src, dst): + """ + Rename a file to a destination path and handles situations where the destination exists. + + :param str src: The current file path. + :param str dst: The new file path. + """ + try: + os.rename(src, dst) + except OSError as err: + # Windows specific, renaming a file on an existing path is not possible. + # On Python 3, the best fallback with atomic capabilities we have is os.replace. + if err.errno != errno.EEXIST: + # Every other error is a legitimate exception. + raise + if not hasattr(os, 'replace'): # pragma: no cover + # We should never go on this line. Either we are on Linux and os.rename has succeeded, + # either we are on Windows, and only Python >= 3.4 is supported where os.replace is + # available. + raise RuntimeError('Error: tried to run os_rename on Python < 3.3. ' + 'Certbot supports only Python 3.4 >= on Windows.') + getattr(os, 'replace')(src, dst) + + def readline_with_timeout(timeout, prompt): """ Read user input to return the first line entered, or raise after specified timeout. diff --git a/certbot/reverter.py b/certbot/reverter.py index 5d56615fd..919037358 100644 --- a/certbot/reverter.py +++ b/certbot/reverter.py @@ -576,7 +576,7 @@ class Reverter(object): timestamp = self._checkpoint_timestamp() final_dir = os.path.join(self.config.backup_dir, timestamp) try: - os.rename(self.config.in_progress_dir, final_dir) + compat.os_rename(self.config.in_progress_dir, final_dir) return except OSError: logger.warning("Extreme, unexpected race condition, retrying (%s)", timestamp) diff --git a/certbot/storage.py b/certbot/storage.py index c16ea35b8..4b8110072 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -14,6 +14,7 @@ import six import certbot from certbot import cli +from certbot import compat from certbot import constants from certbot import crypto_util from certbot import errors @@ -188,7 +189,7 @@ def update_configuration(lineagename, archive_dir, target, cli_config): # Save only the config items that are relevant to renewal values = relevant_values(vars(cli_config.namespace)) write_renewal_config(config_filename, temp_filename, archive_dir, target, values) - os.rename(temp_filename, config_filename) + compat.os_rename(temp_filename, config_filename) return configobj.ConfigObj(config_filename) diff --git a/certbot/tests/compat_test.py b/certbot/tests/compat_test.py new file mode 100644 index 000000000..552aa5645 --- /dev/null +++ b/certbot/tests/compat_test.py @@ -0,0 +1,21 @@ +"""Tests for certbot.compat.""" +import os + +from certbot import compat +import certbot.tests.util as test_util + +class OsReplaceTest(test_util.TempDirTestCase): + """Test to ensure consistent behavior of os_rename method""" + + def test_os_rename_to_existing_file(self): + """Ensure that os_rename will effectively rename src into dst for all platforms.""" + src = os.path.join(self.tempdir, 'src') + dst = os.path.join(self.tempdir, 'dst') + open(src, 'w').close() + open(dst, 'w').close() + + # On Windows, a direct call to os.rename will fail because dst already exists. + compat.os_rename(src, dst) + + self.assertFalse(os.path.exists(src)) + self.assertTrue(os.path.exists(dst)) diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index 999c6225c..d04e3c641 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -356,7 +356,7 @@ class TestFullCheckpointsReverter(test_util.ConfigTestCase): self.assertRaises( errors.ReverterError, self.reverter.finalize_checkpoint, "Title") - @mock.patch("certbot.reverter.os.rename") + @mock.patch("certbot.reverter.compat.os_rename") def test_finalize_checkpoint_no_rename_directory(self, mock_rename): self.reverter.add_to_checkpoint(self.sets[0], "perm save") -- cgit v1.2.3 From e6e323e3ffd8d74251fa9ad46a6eed1bcffbfe38 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 7 Nov 2018 16:49:13 +0100 Subject: Update Lexicon to correct use of HTTP proxy on OVH provider (#6479) This PR update requirement of Lexicon to 2.7.14 on OVH plugin, to allow HTTP proxy to be used correctly when underlying OVH provider is invoked. * Update Lexicon to correct use of HTTP proxy on OVH provider * Update dev_constraints.txt * Update CHANGELOG.md --- CHANGELOG.md | 1 + certbot-dns-ovh/setup.py | 2 +- tools/dev_constraints.txt | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df95c2d36..728c938fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Warn when using deprecated acme.challenges.TLSSNI01 * Log warning about TLS-SNI deprecation in Certbot * Stop preferring TLS-SNI in the Apache, Nginx, and standalone plugins +* OVH DNS plugin now relies on Lexicon>=2.7.14 to support HTTP proxies ### Fixed diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 1f3acbf62..89ff317d9 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -11,7 +11,7 @@ version = '0.28.0.dev0' install_requires = [ 'acme>=0.21.1', 'certbot>=0.21.1', - 'dns-lexicon>=2.7.3', # Correct OVH integration tests + 'dns-lexicon>=2.7.14', # Correct proxy use on OVH provider 'mock', 'setuptools', 'zope.interface', diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 00ecee03e..380d49cb3 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -12,7 +12,7 @@ botocore==1.7.41 cloudflare==1.5.1 coverage==4.4.2 decorator==4.1.2 -dns-lexicon==2.7.3 +dns-lexicon==2.7.14 dnspython==1.15.0 docutils==0.14 execnet==1.5.0 -- cgit v1.2.3 From f3ff548a413a699760ab2239b6e11d65e083d540 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 7 Nov 2018 13:02:25 -0800 Subject: Update changelog for 0.28.0 release. --- CHANGELOG.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 728c938fb..2f0fd40dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). -## 0.28.0 - master +## 0.28.0 - 2018-11-7 ### Added @@ -18,6 +18,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Log warning about TLS-SNI deprecation in Certbot * Stop preferring TLS-SNI in the Apache, Nginx, and standalone plugins * OVH DNS plugin now relies on Lexicon>=2.7.14 to support HTTP proxies +* Default time the Linode plugin waits for DNS changes to propogate is now 1200 seconds. ### Fixed @@ -27,6 +28,30 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Stop caching the results of ipv6_info in http01.py * Test fix for Route53 plugin to prevent boto3 making outgoing connections. * The grammar used by Augeas parser in Apache plugin was updated to fix various parsing errors. +* The CloudXNS, DNSimple, DNS Made Easy, Gehirn, Linode, LuaDNS, NS1, OVH, and + Sakura Cloud DNS plugins are now compatible with Lexicon 3.0+. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* acme +* certbot +* certbot-apache +* certbot-dns-cloudxns +* certbot-dns-dnsimple +* certbot-dns-dnsmadeeasy +* certbot-dns-gehirn +* certbot-dns-linode +* certbot-dns-luadns +* certbot-dns-nsone +* certbot-dns-ovh +* certbot-dns-route53 +* certbot-dns-sakuracloud +* certbot-nginx + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/59?closed=1 ## 0.27.1 - 2018-09-06 -- cgit v1.2.3 From c1300a8e1b5b3d7a485895f9a9abe46d582b2975 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 7 Nov 2018 13:22:57 -0800 Subject: Release 0.28.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 28 ++++++++++----------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 22 ++++++++-------- letsencrypt-auto | 28 ++++++++++----------- letsencrypt-auto-source/certbot-auto.asc | 16 ++++++------ letsencrypt-auto-source/letsencrypt-auto | 26 +++++++++---------- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 +++++++++--------- 26 files changed, 91 insertions(+), 91 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 85492d9a3..89337ad71 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.28.0.dev0' +version = '0.28.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index aa5908f9e..805e1ff5e 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index 076c45e39..fe87317a7 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.27.1" +LE_AUTO_VERSION="0.28.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -195,7 +195,7 @@ if [ "$1" = "--cb-auto-has-root" ]; then else SetRootAuthMechanism if [ -n "$SUDO" ]; then - echo "Requesting to rerun $0 with root privileges..." + say "Requesting to rerun $0 with root privileges..." $SUDO "$0" --cb-auto-has-root "$@" exit 0 fi @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.27.1 \ - --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ - --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a -acme==0.27.1 \ - --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ - --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 -certbot-apache==0.27.1 \ - --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ - --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 -certbot-nginx==0.27.1 \ - --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ - --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f +certbot==0.28.0 \ + --hash=sha256:f2f7c816acd695cbcda713a779b0db2b08e9de407146b46e1c4ef5561e0f5417 \ + --hash=sha256:31e3e2ee2a25c009a621c59ac9182f85d937a897c7bd1d47d0e01f3c712a090a +acme==0.28.0 \ + --hash=sha256:d3a564031155fece3f6edce8c5246d4cf34be0977b4c7c5ce84e86c68601c895 \ + --hash=sha256:bf7c2f1c24a26ab5b9fce3a6abca1d74a5914d46919649ae00ad5817db62bb85 +certbot-apache==0.28.0 \ + --hash=sha256:a57d7bac4f13ae5ecea4f4cbd479d381c02316c4832b25ab5a9d7c6826166370 \ + --hash=sha256:3f93f5de4a548e973c493a6cac5eeeb3dbbcae2988b61299ea0727d04a00f5bb +certbot-nginx==0.28.0 \ + --hash=sha256:1822a65910f0801087fa20a3af3fc94f878f93e0f11809483bb5387df861e296 \ + --hash=sha256:426fb403b0a7b203629f4e350a862cbc3bc1f69936fdab8ec7eafe0d8a3b5ddb UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 8dac3e047..bdda1bcde 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index b823cf98f..4fa1cd5ec 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 3daf933cb..a1083f884 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index feb2a0d58..7d355f5e4 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index b6efcf360..6f6e9181f 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index c268eaa8f..1a099d201 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index fc147f85c..1cec98e24 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 86d36bcb3..696de488d 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 1d196403f..67592de76 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index a5c06d90e..38d2f144b 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 474093a5b..fa4bde85d 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 89ff317d9..ea2a9aa9f 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index c009ef032..b1c1cd0ec 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 2bae0c3d0..c42f976ea 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 9f8bfbbdb..3f54bb96f 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 3c8a66ee5..d292fba31 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index ab23926c9..eecca88e0 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.28.0.dev0' +__version__ = '0.28.0' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 4ed9f0731..d26da361b 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -24,7 +24,7 @@ obtain, install, and renew certificates: manage certificates: certificates Display information about certificates you have from Certbot - revoke Revoke a certificate (supply --cert-path) + revoke Revoke a certificate (supply --cert-path or --cert-name) delete Delete a certificate manage your account with Let's Encrypt: @@ -108,7 +108,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.27.1 + "". (default: CertbotACMEClient/0.28.0 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the @@ -261,7 +261,8 @@ manage: delete Clean up all files related to a certificate renew Renew all certificates (or one specified with --cert- name) - revoke Revoke a certificate specified with --cert-path + revoke Revoke a certificate specified with --cert-path or + --cert-name update_symlinks Recreate symlinks in your /etc/letsencrypt/live/ directory @@ -475,10 +476,9 @@ apache: Apache Web Server plugin - Beta --apache-enmod APACHE_ENMOD - Path to the Apache 'a2enmod' binary (default: a2enmod) + Path to the Apache 'a2enmod' binary (default: None) --apache-dismod APACHE_DISMOD - Path to the Apache 'a2dismod' binary (default: - a2dismod) + Path to the Apache 'a2dismod' binary (default: None) --apache-le-vhost-ext APACHE_LE_VHOST_EXT SSL vhost configuration extension (default: -le- ssl.conf) @@ -492,16 +492,16 @@ apache: /var/log/apache2) --apache-challenge-location APACHE_CHALLENGE_LOCATION Directory path for challenge configuration (default: - /etc/apache2) + /etc/apache2/other) --apache-handle-modules APACHE_HANDLE_MODULES Let installer handle enabling required modules for you - (Only Ubuntu/Debian currently) (default: True) + (Only Ubuntu/Debian currently) (default: False) --apache-handle-sites APACHE_HANDLE_SITES Let installer handle enabling sites for you (Only - Ubuntu/Debian currently) (default: True) + Ubuntu/Debian currently) (default: False) --apache-ctl APACHE_CTL Full path to Apache control script (default: - apache2ctl) + apachectl) certbot-route53:auth: Obtain certificates using a DNS TXT record (if you are using AWS Route53 @@ -602,7 +602,7 @@ dns-linode: --dns-linode-propagation-seconds DNS_LINODE_PROPAGATION_SECONDS The number of seconds to wait for DNS to propagate before asking the ACME server to verify the DNS - record. (default: 960) + record. (default: 1200) --dns-linode-credentials DNS_LINODE_CREDENTIALS Linode credentials INI file. (default: None) diff --git a/letsencrypt-auto b/letsencrypt-auto index 076c45e39..fe87317a7 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.27.1" +LE_AUTO_VERSION="0.28.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -195,7 +195,7 @@ if [ "$1" = "--cb-auto-has-root" ]; then else SetRootAuthMechanism if [ -n "$SUDO" ]; then - echo "Requesting to rerun $0 with root privileges..." + say "Requesting to rerun $0 with root privileges..." $SUDO "$0" --cb-auto-has-root "$@" exit 0 fi @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.27.1 \ - --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ - --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a -acme==0.27.1 \ - --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ - --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 -certbot-apache==0.27.1 \ - --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ - --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 -certbot-nginx==0.27.1 \ - --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ - --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f +certbot==0.28.0 \ + --hash=sha256:f2f7c816acd695cbcda713a779b0db2b08e9de407146b46e1c4ef5561e0f5417 \ + --hash=sha256:31e3e2ee2a25c009a621c59ac9182f85d937a897c7bd1d47d0e01f3c712a090a +acme==0.28.0 \ + --hash=sha256:d3a564031155fece3f6edce8c5246d4cf34be0977b4c7c5ce84e86c68601c895 \ + --hash=sha256:bf7c2f1c24a26ab5b9fce3a6abca1d74a5914d46919649ae00ad5817db62bb85 +certbot-apache==0.28.0 \ + --hash=sha256:a57d7bac4f13ae5ecea4f4cbd479d381c02316c4832b25ab5a9d7c6826166370 \ + --hash=sha256:3f93f5de4a548e973c493a6cac5eeeb3dbbcae2988b61299ea0727d04a00f5bb +certbot-nginx==0.28.0 \ + --hash=sha256:1822a65910f0801087fa20a3af3fc94f878f93e0f11809483bb5387df861e296 \ + --hash=sha256:426fb403b0a7b203629f4e350a862cbc3bc1f69936fdab8ec7eafe0d8a3b5ddb UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 747d98e2d..57745758b 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAluRtuUACgkQTRfJlc2X -dfIvhgf7BrKDo9wjHU8Yb2h1O63OJmoYSQMqM4Q44OVkTTjHQZgDYrOflbegq9g+ -nxxOcMakiPTxvefZOecczKGTZZ/S+A/w5kH/9vJbxW0277iNnYsj1G59m1UPNzgn -ECFL5AUKhl/RF3NWSpe2XhGA7ybls8LAidwxeS3b3nXNeuXIspKd84AIAqaWlpOa -I16NhJsU8VOq6I5RCgkx4WgmmUhCmzjLbYDH7rjj1dehCZa0Y63mlMdTKKs4BJSk -AtSVVV6nTupZdHPJtpQ1RxcT6iTy8Nr13cVuKnluui7KZ/uktOdB0H1o5kuWchvm -8/oqLVSfoqjhU6Fn/11Af+iCnpICUw== -=QRnC +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlvjV5wACgkQTRfJlc2X +dfKkRwf+MJ/Yo5ix7rxGMoliJl3GUUC2KvuYxObvbsAZW69Zl4aZVNeUP3Pe/EZj +zJlSMuiCPeTMmmr0+q78dk5Qk0vf+9D5qSQyy2U+RvPvX6z1PfaFXwjETwOEhE4i +7pABP4m/rIhlZbh336gou4XZK8sXsKHXBLQEyqmzPm6YFZ+5vowIoEinrN73PBuq +rgvoTFKi2NTjYNkQffYUeCIgO0pXlaOa8hkaupqoejHHEjjiXS2C9m0gAT2Wk2cO +zya5WQNcCCLWy/ChhPE2M7yRSpwqrszsHP0qo7QGL8vvsdXvNeJ7vwpAlq/9aipg +PpzSXy/ek8YAgApaj8+/w4OfdDhQ4Q== +=1hD2 -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 740cb9c73..fe87317a7 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.28.0.dev0" +LE_AUTO_VERSION="0.28.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.27.1 \ - --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ - --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a -acme==0.27.1 \ - --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ - --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 -certbot-apache==0.27.1 \ - --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ - --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 -certbot-nginx==0.27.1 \ - --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ - --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f +certbot==0.28.0 \ + --hash=sha256:f2f7c816acd695cbcda713a779b0db2b08e9de407146b46e1c4ef5561e0f5417 \ + --hash=sha256:31e3e2ee2a25c009a621c59ac9182f85d937a897c7bd1d47d0e01f3c712a090a +acme==0.28.0 \ + --hash=sha256:d3a564031155fece3f6edce8c5246d4cf34be0977b4c7c5ce84e86c68601c895 \ + --hash=sha256:bf7c2f1c24a26ab5b9fce3a6abca1d74a5914d46919649ae00ad5817db62bb85 +certbot-apache==0.28.0 \ + --hash=sha256:a57d7bac4f13ae5ecea4f4cbd479d381c02316c4832b25ab5a9d7c6826166370 \ + --hash=sha256:3f93f5de4a548e973c493a6cac5eeeb3dbbcae2988b61299ea0727d04a00f5bb +certbot-nginx==0.28.0 \ + --hash=sha256:1822a65910f0801087fa20a3af3fc94f878f93e0f11809483bb5387df861e296 \ + --hash=sha256:426fb403b0a7b203629f4e350a862cbc3bc1f69936fdab8ec7eafe0d8a3b5ddb UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index b717e359b..33f5b2c00 100644 Binary files a/letsencrypt-auto-source/letsencrypt-auto.sig and b/letsencrypt-auto-source/letsencrypt-auto.sig differ diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index b9cd42694..401a8e25c 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.27.1 \ - --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ - --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a -acme==0.27.1 \ - --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ - --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 -certbot-apache==0.27.1 \ - --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ - --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 -certbot-nginx==0.27.1 \ - --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ - --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f +certbot==0.28.0 \ + --hash=sha256:f2f7c816acd695cbcda713a779b0db2b08e9de407146b46e1c4ef5561e0f5417 \ + --hash=sha256:31e3e2ee2a25c009a621c59ac9182f85d937a897c7bd1d47d0e01f3c712a090a +acme==0.28.0 \ + --hash=sha256:d3a564031155fece3f6edce8c5246d4cf34be0977b4c7c5ce84e86c68601c895 \ + --hash=sha256:bf7c2f1c24a26ab5b9fce3a6abca1d74a5914d46919649ae00ad5817db62bb85 +certbot-apache==0.28.0 \ + --hash=sha256:a57d7bac4f13ae5ecea4f4cbd479d381c02316c4832b25ab5a9d7c6826166370 \ + --hash=sha256:3f93f5de4a548e973c493a6cac5eeeb3dbbcae2988b61299ea0727d04a00f5bb +certbot-nginx==0.28.0 \ + --hash=sha256:1822a65910f0801087fa20a3af3fc94f878f93e0f11809483bb5387df861e296 \ + --hash=sha256:426fb403b0a7b203629f4e350a862cbc3bc1f69936fdab8ec7eafe0d8a3b5ddb -- cgit v1.2.3 From 22858c6025c2697ea7842f97a91e366cbac8293a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 7 Nov 2018 13:22:59 -0800 Subject: Bump version to 0.29.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 89337ad71..ad70c2947 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.28.0' +version = '0.29.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 805e1ff5e..e6f6f1e23 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index bdda1bcde..bfbbe0625 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 4fa1cd5ec..d615fa999 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index a1083f884..f9880270a 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 7d355f5e4..a9d46d128 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 6f6e9181f..ac7bb1090 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 1a099d201..ab944d40d 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 1cec98e24..94ca74761 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 696de488d..aa1afdc93 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 67592de76..4c2571f96 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 38d2f144b..d0735d1ec 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index fa4bde85d..aebff5304 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index ea2a9aa9f..68ede8006 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index b1c1cd0ec..914bfb6e6 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index c42f976ea..6318cbb81 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 3f54bb96f..5af4c8a00 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index d292fba31..0908c4c52 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index eecca88e0..f6b7defbd 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.28.0' +__version__ = '0.29.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index fe87317a7..b1a05f6e6 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.28.0" +LE_AUTO_VERSION="0.29.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates -- cgit v1.2.3 From 63e0f5678457b78cfee7fa7f8337a4b8f1809fd8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 7 Nov 2018 15:56:29 -0800 Subject: update changelog for 0.29.0 --- CHANGELOG.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f0fd40dc..a25890929 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,29 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.29.0 - master + +### Added + +* + +### Changed + +* + +### Fixed + +* + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/62?closed=1 + ## 0.28.0 - 2018-11-7 ### Added -- cgit v1.2.3 From 3d0e16ece3762d5bfe345fde0af286e26c54bacb Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 8 Nov 2018 02:16:16 +0100 Subject: [Windows|Unix] Rewrite bash scripts for tests into python (#6435) Certbot relies heavily on bash scripts to deploy a development environment and to execute tests. This is fine for Linux systems, including Travis, but problematic for Windows machines. This PR converts all theses scripts into Python, to make them platform independant. As a consequence, tox-win.ini is not needed anymore, and tox can be run indifferently on Windows or on Linux using a common tox.ini. AppVeyor is updated accordingly to execute tests for acme, certbot and all dns plugins. Other tests are not executed as they are for Docker, unsupported Apache/Nginx/Postfix plugins (for now) or not relevant for Windows (explicit Linux distribution tests or pylint). Another PR will be done on certbot website to update how a dev environment can be set up. * Replace several shell scripts by python equivalent. * Correction on tox coverage * Extend usage of new python scripts * Various corrections * Replace venv construction bash scripts by python equivalents * Update tox.ini * Unicode lines to compare files * Put modifications on letsencrypt-auto-source instead of generated scripts * Add executable permissions for Linux. * Merge tox win tests into main tox * Skip lock_test on Windows * Correct appveyor config * Update appveyor.yml * Explicit coverage py27 or py37 * Avoid to cover non supported certbot plugins on Windows * Update tox.ini * Remove specific warnings during CI * No cover on a debug code for tests only. * Update documentation and help script on venv/venv3.py * Customize help message for Windows * Quote correctly executable path with potential spaces in it. * Copy pipstrap from upstream --- .travis.yml | 4 +- Dockerfile-dev | 2 +- appveyor.yml | 25 +++-- certbot-compatibility-test/Dockerfile | 5 +- certbot-postfix/README.rst | 2 +- certbot/tests/util.py | 13 ++- docs/contributing.rst | 8 +- letsencrypt-auto-source/letsencrypt-auto | 11 +- .../pieces/bootstrappers/arch_common.sh | 2 +- letsencrypt-auto-source/pieces/pipstrap.py | 10 +- tests/letstest/scripts/test_apache2.sh | 2 +- tests/letstest/scripts/test_tox.sh | 2 +- tests/lock_test.py | 9 +- tests/modification-check.py | 124 +++++++++++++++++++++ tests/modification-check.sh | 59 ---------- tools/_venv_common.py | 67 +++++++++++ tools/_venv_common.sh | 26 ----- tools/install_and_test.py | 58 ++++++++++ tools/install_and_test.sh | 29 ----- tools/merge_requirements.py | 16 +-- tools/pip_install.py | 90 +++++++++++++++ tools/pip_install.sh | 44 -------- tools/pip_install_editable.py | 19 ++++ tools/pip_install_editable.sh | 10 -- tools/readlink.py | 7 +- tools/venv.py | 58 ++++++++++ tools/venv.sh | 34 ------ tools/venv3.py | 53 +++++++++ tools/venv3.sh | 33 ------ tox-win.ini | 13 --- tox.cover.py | 85 ++++++++++++++ tox.cover.sh | 72 ------------ tox.ini | 28 +++-- 33 files changed, 643 insertions(+), 377 deletions(-) create mode 100755 tests/modification-check.py delete mode 100755 tests/modification-check.sh create mode 100755 tools/_venv_common.py delete mode 100755 tools/_venv_common.sh create mode 100755 tools/install_and_test.py delete mode 100755 tools/install_and_test.sh create mode 100755 tools/pip_install.py delete mode 100755 tools/pip_install.sh create mode 100755 tools/pip_install_editable.py delete mode 100755 tools/pip_install_editable.sh create mode 100755 tools/venv.py delete mode 100755 tools/venv.sh create mode 100755 tools/venv3.py delete mode 100755 tools/venv3.sh delete mode 100644 tox-win.ini create mode 100755 tox.cover.py delete mode 100755 tox.cover.sh diff --git a/.travis.yml b/.travis.yml index acdf365bd..2b8eafc13 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ matrix: sudo: required services: docker - python: "2.7" - env: TOXENV=cover FYI="this also tests py27" + env: TOXENV=py27-cover FYI="py27 tests + code coverage" - sudo: required env: TOXENV=nginx_compat services: docker @@ -95,7 +95,7 @@ script: - travis_retry tox - '[ -z "${BOULDER_INTEGRATION+x}" ] || (travis_retry tests/boulder-fetch.sh && tests/tox-boulder-integration.sh)' -after_success: '[ "$TOXENV" == "cover" ] && codecov' +after_success: '[ "$TOXENV" == "py27-cover" ] && codecov' notifications: email: false diff --git a/Dockerfile-dev b/Dockerfile-dev index 9e35ebec8..1ab56e081 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -16,6 +16,6 @@ RUN apt-get update && \ /tmp/* \ /var/tmp/* -RUN VENV_NAME="../venv" tools/venv.sh +RUN VENV_NAME="../venv" python tools/venv.py ENV PATH /opt/certbot/venv/bin:$PATH diff --git a/appveyor.yml b/appveyor.yml index 796070081..725ecfbff 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,8 +1,17 @@ -image: - # => Windows Server 2012 R2 - - Visual Studio 2015 - # => Windows Server 2016 - - Visual Studio 2017 +environment: + matrix: + - FYI: Python 3.4 on Windows Server 2012 R2 + TOXENV: py34 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 + - FYI: Python 3.4 on Windows Server 2016 + TOXENV: py34 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + - FYI: Python 3.5 on Windows Server 2016 + TOXENV: py35 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + - FYI: Python 3.7 on Windows Server 2016 + code coverage + TOXENV: py37-cover + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 branches: only: @@ -14,6 +23,7 @@ install: # Use Python 3.7 by default - "SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%" # Check env + - "echo %APPVEYOR_BUILD_WORKER_IMAGE%" - "python --version" # Upgrade pip to avoid warnings - "python -m pip install --upgrade pip" @@ -23,7 +33,8 @@ install: build: off test_script: - - tox -c tox-win.ini -e py34,py35,py36,py37-cover + # Test env is set by TOXENV env variable + - tox on_success: - - codecov + - if exist .coverage codecov diff --git a/certbot-compatibility-test/Dockerfile b/certbot-compatibility-test/Dockerfile index 803b4a1b9..cbbdf7193 100644 --- a/certbot-compatibility-test/Dockerfile +++ b/certbot-compatibility-test/Dockerfile @@ -14,7 +14,7 @@ RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only # the above is not likely to change, so by putting it further up the # Dockerfile we make sure we cache as much as possible -COPY setup.py README.rst CHANGELOG.md MANIFEST.in linter_plugin.py tox.cover.sh tox.ini .pylintrc /opt/certbot/src/ +COPY setup.py README.rst CHANGELOG.md MANIFEST.in linter_plugin.py tox.cover.py tox.ini .pylintrc /opt/certbot/src/ # all above files are necessary for setup.py, however, package source # code directory has to be copied separately to a subdirectory... @@ -35,7 +35,8 @@ RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv && \ /opt/certbot/venv/bin/pip install -U setuptools && \ /opt/certbot/venv/bin/pip install -U pip ENV PATH /opt/certbot/venv/bin:$PATH -RUN /opt/certbot/src/tools/pip_install_editable.sh \ +RUN /opt/certbot/venv/bin/python \ + /opt/certbot/src/tools/pip_install_editable.py \ /opt/certbot/src/acme \ /opt/certbot/src \ /opt/certbot/src/certbot-apache \ diff --git a/certbot-postfix/README.rst b/certbot-postfix/README.rst index e6367e365..1ae9cb980 100644 --- a/certbot-postfix/README.rst +++ b/certbot-postfix/README.rst @@ -7,7 +7,7 @@ feature requests for this plugin. To install this plugin, in the root of this repo, run:: - ./tools/venv.sh + python tools/venv.py source venv/bin/activate You can use this installer with any `authenticator plugin diff --git a/certbot/tests/util.py b/certbot/tests/util.py index 822597dd4..8c5db2c2f 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -328,15 +328,16 @@ class TempDirTestCase(unittest.TestCase): def tearDown(self): """Execute after test""" - # Then we have various files which are not correctly closed at the time of tearDown. - # On Windows, it is visible for the same reasons as above. + # On Windows we have various files which are not correctly closed at the time of tearDown. # For know, we log them until a proper file close handling is written. + # Useful for development only, so no warning when we are on a CI process. def onerror_handler(_, path, excinfo): """On error handler""" - message = ('Following error occurred when deleting the tempdir {0}' - ' for path {1} during tearDown process: {2}' - .format(self.tempdir, path, str(excinfo))) - warnings.warn(message) + if not os.environ.get('APPVEYOR'): # pragma: no cover + message = ('Following error occurred when deleting the tempdir {0}' + ' for path {1} during tearDown process: {2}' + .format(self.tempdir, path, str(excinfo))) + warnings.warn(message) shutil.rmtree(self.tempdir, onerror=onerror_handler) class ConfigTestCase(TempDirTestCase): diff --git a/docs/contributing.rst b/docs/contributing.rst index 58db251d4..ead4d7e2b 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -38,13 +38,13 @@ Certbot. cd certbot ./certbot-auto --debug --os-packages-only - tools/venv.sh + python tools/venv.py -If you have Python3 available and want to use it, run the ``venv3.sh`` script. +If you have Python3 available and want to use it, run the ``venv3.py`` script. .. code-block:: shell - tools/venv3.sh + python tools/venv3.py .. note:: You may need to repeat this when Certbot's dependencies change or when a new plugin is introduced. @@ -353,7 +353,7 @@ Steps: 1. Write your code! 2. Make sure your environment is set up properly and that you're in your - virtualenv. You can do this by running ``./tools/venv.sh``. + virtualenv. You can do this by running ``pip tools/venv.py``. (this is a **very important** step) 3. Run ``tox -e lint`` to check for pylint errors. Fix any errors. 4. Run ``tox --skip-missing-interpreters`` to run the entire test suite diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index b1a05f6e6..12be26e19 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -594,7 +594,7 @@ BootstrapArchCommon() { # # "python-virtualenv" is Python3, but "python2-virtualenv" provides # only "virtualenv2" binary, not "virtualenv" necessary in - # ./tools/_venv_common.sh + # ./tools/_venv_common.py deps=" python2 @@ -1260,7 +1260,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info +from sys import exit, version_info, executable from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -1272,7 +1272,7 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 5, 1 +__version__ = 2, 0, 0 PIP_VERSION = '9.0.1' DEFAULT_INDEX_BASE = 'https://pypi.python.org' @@ -1365,7 +1365,7 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output(['pip', '--version']) + pip_version = StrictVersion(check_output([executable, '-m', 'pip', '--version']) .decode('utf-8').split()[1]) min_pip_version = StrictVersion(PIP_VERSION) if pip_version >= min_pip_version: @@ -1378,7 +1378,7 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + + check_output('{0} -m pip install --no-index --no-deps -U '.format(quote(executable)) + # Disable cache since we're not using it and it otherwise # sometimes throws permission warnings: ('--no-cache-dir ' if has_pip_cache else '') + @@ -1397,7 +1397,6 @@ def main(): if __name__ == '__main__': exit(main()) - UNLIKELY_EOF # ------------------------------------------------------------------------- # Set PATH so pipstrap upgrades the right (v)env: diff --git a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh index 5759336c5..c55527590 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh @@ -8,7 +8,7 @@ BootstrapArchCommon() { # # "python-virtualenv" is Python3, but "python2-virtualenv" provides # only "virtualenv2" binary, not "virtualenv" necessary in - # ./tools/_venv_common.sh + # ./tools/_venv_common.py deps=" python2 diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index d55d5bceb..f21d36657 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -45,7 +45,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info +from sys import exit, version_info, executable from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -57,7 +57,7 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 5, 1 +__version__ = 2, 0, 0 PIP_VERSION = '9.0.1' DEFAULT_INDEX_BASE = 'https://pypi.python.org' @@ -150,7 +150,7 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output(['pip', '--version']) + pip_version = StrictVersion(check_output([executable, '-m', 'pip', '--version']) .decode('utf-8').split()[1]) min_pip_version = StrictVersion(PIP_VERSION) if pip_version >= min_pip_version: @@ -163,7 +163,7 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + + check_output('{0} -m pip install --no-index --no-deps -U '.format(quote(executable)) + # Disable cache since we're not using it and it otherwise # sometimes throws permission warnings: ('--no-cache-dir ' if has_pip_cache else '') + @@ -181,4 +181,4 @@ def main(): if __name__ == '__main__': - exit(main()) + exit(main()) \ No newline at end of file diff --git a/tests/letstest/scripts/test_apache2.sh b/tests/letstest/scripts/test_apache2.sh index 6b5d63c80..4036e6efa 100755 --- a/tests/letstest/scripts/test_apache2.sh +++ b/tests/letstest/scripts/test_apache2.sh @@ -45,7 +45,7 @@ if [ $? -ne 0 ] ; then exit 1 fi -tools/_venv_common.sh -e acme[dev] -e .[dev,docs] -e certbot-apache +python tools/_venv_common.py -e acme[dev] -e .[dev,docs] -e certbot-apache sudo venv/bin/certbot -v --debug --text --agree-dev-preview --agree-tos \ --renew-by-default --redirect --register-unsafely-without-email \ --domain $PUBLIC_HOSTNAME --server $BOULDER_URL diff --git a/tests/letstest/scripts/test_tox.sh b/tests/letstest/scripts/test_tox.sh index 84e4bcd22..bb9126673 100755 --- a/tests/letstest/scripts/test_tox.sh +++ b/tests/letstest/scripts/test_tox.sh @@ -14,5 +14,5 @@ VENV_BIN=${VENV_PATH}/bin "$LEA_PATH/letsencrypt-auto" --os-packages-only cd letsencrypt -./tools/venv.sh +python tools/venv.py venv/bin/tox -e py27 diff --git a/tests/lock_test.py b/tests/lock_test.py index b01cc5d58..0266cf029 100644 --- a/tests/lock_test.py +++ b/tests/lock_test.py @@ -1,4 +1,6 @@ """Tests to ensure the lock order is preserved.""" +from __future__ import print_function + import atexit import functools import logging @@ -235,4 +237,9 @@ def log_output(level, out, err): if __name__ == "__main__": - main() + if os.name != 'nt': + main() + else: + print( + 'Warning: lock_test cannot be executed on Windows, ' + 'as it relies on a Nginx distribution for Linux.') diff --git a/tests/modification-check.py b/tests/modification-check.py new file mode 100755 index 000000000..e00994b04 --- /dev/null +++ b/tests/modification-check.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import os +import subprocess +import sys +import tempfile +import shutil +try: + from urllib.request import urlretrieve +except ImportError: + from urllib import urlretrieve + +def find_repo_path(): + return os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + +# We do not use filecmp.cmp to take advantage of universal newlines +# handling in open() for Python 3.x and be insensitive to CRLF/LF when run on Windows. +# As a consequence, this function will not work correctly if executed by Python 2.x on Windows. +# But it will work correctly on Linux for any version, because every file tested will be LF. +def compare_files(path_1, path_2): + l1 = l2 = True + with open(path_1, 'r') as f1, open(path_2, 'r') as f2: + line = 1 + while l1 and l2: + line += 1 + l1 = f1.readline() + l2 = f2.readline() + if l1 != l2: + print('---') + print(( + 'While comparing {0} (1) and {1} (2), a difference was found at line {2}:' + .format(os.path.basename(path_1), os.path.basename(path_2), line))) + print('(1): {0}'.format(repr(l1))) + print('(2): {0}'.format(repr(l2))) + print('---') + return False + + return True + +def validate_scripts_content(repo_path, temp_cwd): + errors = False + + if not compare_files( + os.path.join(repo_path, 'certbot-auto'), + os.path.join(repo_path, 'letsencrypt-auto')): + print('Root certbot-auto and letsencrypt-auto differ.') + errors = True + else: + shutil.copyfile( + os.path.join(repo_path, 'certbot-auto'), + os.path.join(temp_cwd, 'local-auto')) + shutil.copy(os.path.normpath(os.path.join( + repo_path, + 'letsencrypt-auto-source/pieces/fetch.py')), temp_cwd) + + # Compare file against current version in the target branch + branch = os.environ.get('TRAVIS_BRANCH', 'master') + url = ( + 'https://raw.githubusercontent.com/certbot/certbot/{0}/certbot-auto' + .format(branch)) + urlretrieve(url, os.path.join(temp_cwd, 'certbot-auto')) + + if compare_files( + os.path.join(temp_cwd, 'certbot-auto'), + os.path.join(temp_cwd, 'local-auto')): + print('Root *-auto were unchanged') + else: + # Compare file against the latest released version + latest_version = subprocess.check_output( + [sys.executable, 'fetch.py', '--latest-version'], cwd=temp_cwd) + subprocess.call( + [sys.executable, 'fetch.py', '--le-auto-script', + 'v{0}'.format(latest_version.decode().strip())], cwd=temp_cwd) + if compare_files( + os.path.join(temp_cwd, 'letsencrypt-auto'), + os.path.join(temp_cwd, 'local-auto')): + print('Root *-auto were updated to the latest version.') + else: + print('Root *-auto have unexpected changes.') + errors = True + + return errors + +def main(): + repo_path = find_repo_path() + temp_cwd = tempfile.mkdtemp() + errors = False + + try: + errors = validate_scripts_content(repo_path, temp_cwd) + + shutil.copyfile( + os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')), + os.path.join(temp_cwd, 'original-lea') + ) + subprocess.call([sys.executable, os.path.normpath(os.path.join( + repo_path, 'letsencrypt-auto-source/build.py'))]) + shutil.copyfile( + os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')), + os.path.join(temp_cwd, 'build-lea') + ) + shutil.copyfile( + os.path.join(temp_cwd, 'original-lea'), + os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')) + ) + + if not compare_files( + os.path.join(temp_cwd, 'original-lea'), + os.path.join(temp_cwd, 'build-lea')): + print('Script letsencrypt-auto-source/letsencrypt-auto ' + 'doesn\'t match output of build.py.') + errors = True + else: + print('Script letsencrypt-auto-source/letsencrypt-auto matches output of build.py.') + finally: + shutil.rmtree(temp_cwd) + + return errors + +if __name__ == '__main__': + if main(): + sys.exit(1) diff --git a/tests/modification-check.sh b/tests/modification-check.sh deleted file mode 100755 index 0145b0228..000000000 --- a/tests/modification-check.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -e - -temp_dir=`mktemp -d` -trap "rm -rf $temp_dir" EXIT - -# cd to repo root -cd $(dirname $(dirname $(readlink -f $0))) -FLAG=false - -if ! cmp -s certbot-auto letsencrypt-auto; then - echo "Root certbot-auto and letsencrypt-auto differ." - FLAG=true -else - cp certbot-auto "$temp_dir/local-auto" - cp letsencrypt-auto-source/pieces/fetch.py "$temp_dir/fetch.py" - cd $temp_dir - - # Compare file against current version in the target branch - BRANCH=${TRAVIS_BRANCH:-master} - URL="https://raw.githubusercontent.com/certbot/certbot/$BRANCH/certbot-auto" - curl -sS $URL > certbot-auto - if cmp -s certbot-auto local-auto; then - echo "Root *-auto were unchanged." - else - # Compare file against the latest released version - python fetch.py --le-auto-script "v$(python fetch.py --latest-version)" - if cmp -s letsencrypt-auto local-auto; then - echo "Root *-auto were updated to the latest version." - else - echo "Root *-auto have unexpected changes." - FLAG=true - fi - fi - cd ~- -fi - -# Compare letsencrypt-auto-source/letsencrypt-auto with output of build.py - -cp letsencrypt-auto-source/letsencrypt-auto ${temp_dir}/original-lea -python letsencrypt-auto-source/build.py -cp letsencrypt-auto-source/letsencrypt-auto ${temp_dir}/build-lea -cp ${temp_dir}/original-lea letsencrypt-auto-source/letsencrypt-auto - -cd $temp_dir - -if ! cmp -s original-lea build-lea; then - echo "letsencrypt-auto-source/letsencrypt-auto doesn't match output of \ -build.py." - FLAG=true -else - echo "letsencrypt-auto-source/letsencrypt-auto matches output of \ -build.py." -fi - -rm -rf $temp_dir - -if $FLAG ; then - exit 1 -fi diff --git a/tools/_venv_common.py b/tools/_venv_common.py new file mode 100755 index 000000000..0c24664b3 --- /dev/null +++ b/tools/_venv_common.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import os +import shutil +import glob +import time +import subprocess +import sys + +def subprocess_with_print(command): + print(command) + subprocess.call(command, shell=True) + +def get_venv_python(venv_path): + python_linux = os.path.join(venv_path, 'bin/python') + python_windows = os.path.join(venv_path, 'Scripts\\python.exe') + if os.path.isfile(python_linux): + return python_linux + if os.path.isfile(python_windows): + return python_windows + + raise ValueError(( + 'Error, could not find python executable in venv path {0}: is it a valid venv ?' + .format(venv_path))) + +def main(venv_name, venv_args, args): + for path in glob.glob('*.egg-info'): + if os.path.isdir(path): + shutil.rmtree(path) + else: + os.remove(path) + + if os.path.isdir(venv_name): + os.rename(venv_name, '{0}.{1}.bak'.format(venv_name, int(time.time()))) + + subprocess_with_print(' '.join([ + sys.executable, '-m', 'virtualenv', '--no-site-packages', '--setuptools', + venv_name, venv_args])) + + python_executable = get_venv_python(venv_name) + + subprocess_with_print(' '.join([ + python_executable, os.path.normpath('./letsencrypt-auto-source/pieces/pipstrap.py')])) + command = [python_executable, os.path.normpath('./tools/pip_install.py')] + command.extend(args) + subprocess_with_print(' '.join(command)) + + if os.path.isdir(os.path.join(venv_name, 'bin')): + # Linux/OSX specific + print('-------------------------------------------------------------------') + print('Please run the following command to activate developer environment:') + print('source {0}/bin/activate'.format(venv_name)) + print('-------------------------------------------------------------------') + elif os.path.isdir(os.path.join(venv_args, 'Scripts')): + # Windows specific + print('---------------------------------------------------------------------------') + print('Please run one of the following commands to activate developer environment:') + print('{0}\\bin\\activate.bat (for Batch)'.format(venv_name)) + print('.\\{0}\\Scripts\\Activate.ps1 (for Powershell)'.format(venv_name)) + print('---------------------------------------------------------------------------') + else: + raise ValueError('Error, directory {0} is not a valid venv.'.format(venv_name)) + +if __name__ == '__main__': + main(os.environ.get('VENV_NAME', 'venv'), os.environ.get('VENV_ARGS', ''), sys.argv[1:]) diff --git a/tools/_venv_common.sh b/tools/_venv_common.sh deleted file mode 100755 index 0f0ff7e28..000000000 --- a/tools/_venv_common.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -xe - -VENV_NAME=${VENV_NAME:-venv} - -# .egg-info directories tend to cause bizarre problems (e.g. `pip -e -# .` might unexpectedly install letshelp-certbot only, in case -# `python letshelp-certbot/setup.py build` has been called -# earlier) -rm -rf *.egg-info - -# virtualenv setup is NOT idempotent: shutil.Error: -# `/home/jakub/dev/letsencrypt/letsencrypt/venv/bin/python2` and -# `venv/bin/python2` are the same file -mv $VENV_NAME "$VENV_NAME.$(date +%s).bak" || true -virtualenv --no-site-packages --setuptools $VENV_NAME $VENV_ARGS -. ./$VENV_NAME/bin/activate - -# Use pipstrap to update Python packaging tools to only update to a well tested -# version and to work around https://github.com/pypa/pip/issues/4817 on older -# systems. -python letsencrypt-auto-source/pieces/pipstrap.py -./tools/pip_install.sh "$@" - -set +x -echo "Please run the following command to activate developer environment:" -echo "source $VENV_NAME/bin/activate" diff --git a/tools/install_and_test.py b/tools/install_and_test.py new file mode 100755 index 000000000..b16181aa5 --- /dev/null +++ b/tools/install_and_test.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# pip installs the requested packages in editable mode and runs unit tests on +# them. Each package is installed and tested in the order they are provided +# before the script moves on to the next package. If CERTBOT_NO_PIN is set not +# set to 1, packages are installed using pinned versions of all of our +# dependencies. See pip_install.py for more information on the versions pinned +# to. +from __future__ import print_function + +import os +import sys +import tempfile +import shutil +import subprocess +import re + +SKIP_PROJECTS_ON_WINDOWS = [ + 'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot'] + +def call_with_print(command, cwd=None): + print(command) + subprocess.call(command, shell=True, cwd=cwd or os.getcwd()) + +def main(args): + if os.environ.get('CERTBOT_NO_PIN') == '1': + command = [sys.executable, '-m', 'pip', '-q', '-e'] + else: + script_dir = os.path.dirname(os.path.abspath(__file__)) + command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')] + + new_args = [] + for arg in args: + if os.name == 'nt' and arg in SKIP_PROJECTS_ON_WINDOWS: + print(( + 'Info: currently {0} is not supported on Windows and will not be tested.' + .format(arg))) + else: + new_args.append(arg) + + for requirement in new_args: + current_command = command[:] + current_command.append(requirement) + call_with_print(' '.join(current_command)) + pkg = re.sub(r'\[\w+\]', '', requirement) + + if pkg == '.': + pkg = 'certbot' + + temp_cwd = tempfile.mkdtemp() + try: + call_with_print(' '.join([ + sys.executable, '-m', 'pytest', '--numprocesses', 'auto', + '--quiet', '--pyargs', pkg.replace('-', '_')]), cwd=temp_cwd) + finally: + shutil.rmtree(temp_cwd) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh deleted file mode 100755 index 819f683aa..000000000 --- a/tools/install_and_test.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh -e -# pip installs the requested packages in editable mode and runs unit tests on -# them. Each package is installed and tested in the order they are provided -# before the script moves on to the next package. If CERTBOT_NO_PIN is set not -# set to 1, packages are installed using pinned versions of all of our -# dependencies. See pip_install.sh for more information on the versions pinned -# to. - -if [ "$CERTBOT_NO_PIN" = 1 ]; then - pip_install="pip install -q -e" -else - pip_install="$(dirname $0)/pip_install_editable.sh" -fi - -temp_cwd=$(mktemp -d) -trap "rm -rf $temp_cwd" EXIT - -set -x -for requirement in "$@" ; do - $pip_install $requirement - pkg=$(echo $requirement | cut -f1 -d\[) # remove any extras such as [dev] - pkg=$(echo "$pkg" | tr - _ ) # convert package names to Python import names - if [ $pkg = "." ]; then - pkg="certbot" - fi - cd "$temp_cwd" - pytest --numprocesses auto --quiet --pyargs $pkg - cd - -done diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py index c8fb95351..ad44a55d0 100755 --- a/tools/merge_requirements.py +++ b/tools/merge_requirements.py @@ -10,7 +10,6 @@ from __future__ import print_function import sys - def read_file(file_path): """Reads in a Python requirements file. @@ -32,17 +31,17 @@ def read_file(file_path): return d -def print_requirements(requirements): - """Prints requirements to stdout. +def output_requirements(requirements): + """Prepare print requirements to stdout. :param dict requirements: mapping from a project to its pinned version """ - print('\n'.join('{0}=={1}'.format(k, v) - for k, v in sorted(requirements.items()))) + return '\n'.join('{0}=={1}'.format(k, v) + for k, v in sorted(requirements.items())) -def merge_requirements_files(*files): +def main(*files): """Merges multiple requirements files together and prints the result. Requirement files specified later in the list take precedence over earlier @@ -54,8 +53,9 @@ def merge_requirements_files(*files): d = {} for f in files: d.update(read_file(f)) - print_requirements(d) + return output_requirements(d) if __name__ == '__main__': - merge_requirements_files(*sys.argv[1:]) + merged_requirements = main(*sys.argv[1:]) + print(merged_requirements) diff --git a/tools/pip_install.py b/tools/pip_install.py new file mode 100755 index 000000000..d09997bf5 --- /dev/null +++ b/tools/pip_install.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set +# to 1, a combination of tools/oldest_constraints.txt, +# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the +# top level of the package's directory is used, otherwise, a combination of +# certbot-auto's requirements file and tools/dev_constraints.txt is used. The +# other file always takes precedence over tools/dev_constraints.txt. If +# CERTBOT_OLDEST is set, this script must be run with `-e ` and +# no other arguments. + +from __future__ import print_function, absolute_import + +import subprocess +import os +import sys +import re +import shutil +import tempfile + +import merge_requirements as merge_module +import readlink + +def find_tools_path(): + return os.path.dirname(readlink.main(__file__)) + +def certbot_oldest_processing(tools_path, args, test_constraints): + if args[0] != '-e' or len(args) != 2: + raise ValueError('When CERTBOT_OLDEST is set, this script must be run ' + 'with a single -e argument.') + # remove any extras such as [dev] + pkg_dir = re.sub(r'\[\w+\]', '', args[1]) + requirements = os.path.join(pkg_dir, 'local-oldest-requirements.txt') + # packages like acme don't have any local oldest requirements + if not os.path.isfile(requirements): + requirements = None + shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints) + + return requirements + +def certbot_normal_processing(tools_path, test_constraints): + repo_path = os.path.dirname(tools_path) + certbot_requirements = os.path.normpath(os.path.join( + repo_path, 'letsencrypt-auto-source/pieces/dependency-requirements.txt')) + with open(certbot_requirements, 'r') as fd: + data = fd.readlines() + with open(test_constraints, 'w') as fd: + for line in data: + search = re.search(r'^(\S*==\S*).*$', line) + if search: + fd.write('{0}{1}'.format(search.group(1), os.linesep)) + +def merge_requirements(tools_path, test_constraints, all_constraints): + merged_requirements = merge_module.main( + os.path.join(tools_path, 'dev_constraints.txt'), + test_constraints + ) + with open(all_constraints, 'w') as fd: + fd.write(merged_requirements) + +def call_with_print(command, cwd=None): + print(command) + subprocess.call(command, shell=True, cwd=cwd or os.getcwd()) + +def main(args): + tools_path = find_tools_path() + working_dir = tempfile.mkdtemp() + try: + test_constraints = os.path.join(working_dir, 'test_constraints.txt') + all_constraints = os.path.join(working_dir, 'all_constraints.txt') + + requirements = None + if os.environ.get('CERTBOT_OLDEST') == '1': + requirements = certbot_oldest_processing(tools_path, args, test_constraints) + else: + certbot_normal_processing(tools_path, test_constraints) + + merge_requirements(tools_path, test_constraints, all_constraints) + if requirements: + call_with_print(' '.join([ + sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints, + '--requirement', requirements])) + + command = [sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints] + command.extend(args) + call_with_print(' '.join(command)) + finally: + shutil.rmtree(working_dir) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/tools/pip_install.sh b/tools/pip_install.sh deleted file mode 100755 index 78e2afa17..000000000 --- a/tools/pip_install.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh -e -# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set -# to 1, a combination of tools/oldest_constraints.txt, -# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the -# top level of the package's directory is used, otherwise, a combination of -# certbot-auto's requirements file and tools/dev_constraints.txt is used. The -# other file always takes precedence over tools/dev_constraints.txt. If -# CERTBOT_OLDEST is set, this script must be run with `-e ` and -# no other arguments. - -# get the root of the Certbot repo -tools_dir=$(dirname $("$(dirname $0)/readlink.py" $0)) -all_constraints=$(mktemp) -test_constraints=$(mktemp) -trap "rm -f $all_constraints $test_constraints" EXIT - -if [ "$CERTBOT_OLDEST" = 1 ]; then - if [ "$1" != "-e" -o "$#" -ne "2" ]; then - echo "When CERTBOT_OLDEST is set, this script must be run with a single -e argument." - exit 1 - fi - pkg_dir=$(echo $2 | cut -f1 -d\[) # remove any extras such as [dev] - requirements="$pkg_dir/local-oldest-requirements.txt" - # packages like acme don't have any local oldest requirements - if [ ! -f "$requirements" ]; then - unset requirements - fi - cp "$tools_dir/oldest_constraints.txt" "$test_constraints" -else - repo_root=$(dirname "$tools_dir") - certbot_requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt" - sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' "$certbot_requirements" > "$test_constraints" -fi - -"$tools_dir/merge_requirements.py" "$tools_dir/dev_constraints.txt" \ - "$test_constraints" > "$all_constraints" - -set -x - -# install the requested packages using the pinned requirements as constraints -if [ -n "$requirements" ]; then - pip install -q --constraint "$all_constraints" --requirement "$requirements" -fi -pip install -q --constraint "$all_constraints" "$@" diff --git a/tools/pip_install_editable.py b/tools/pip_install_editable.py new file mode 100755 index 000000000..bdbdcadc5 --- /dev/null +++ b/tools/pip_install_editable.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +# pip installs packages in editable mode using certbot-auto's requirements file +# as constraints + +from __future__ import absolute_import + +import sys + +import pip_install + +def main(args): + new_args = [] + for arg in args: + new_args.append('-e') + new_args.append(arg) + pip_install.main(new_args) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/tools/pip_install_editable.sh b/tools/pip_install_editable.sh deleted file mode 100755 index 6130bf6e7..000000000 --- a/tools/pip_install_editable.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -e -# pip installs packages in editable mode using certbot-auto's requirements file -# as constraints - -args="" -for requirement in "$@" ; do - args="$args -e $requirement" -done - -"$(dirname $0)/pip_install.sh" $args diff --git a/tools/readlink.py b/tools/readlink.py index 02c74c48d..0199ce184 100755 --- a/tools/readlink.py +++ b/tools/readlink.py @@ -7,7 +7,12 @@ platforms. """ from __future__ import print_function + import os import sys -print(os.path.realpath(sys.argv[1])) +def main(link): + return os.path.realpath(link) + +if __name__ == '__main__': + print(main(sys.argv[1])) diff --git a/tools/venv.py b/tools/venv.py new file mode 100755 index 000000000..849c49119 --- /dev/null +++ b/tools/venv.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# Developer virtualenv setup for Certbot client + +from __future__ import absolute_import + +import os +import subprocess + +import _venv_common + +REQUIREMENTS = [ + '-e acme[dev]', + '-e .[dev,docs]', + '-e certbot-apache', + '-e certbot-dns-cloudflare', + '-e certbot-dns-cloudxns', + '-e certbot-dns-digitalocean', + '-e certbot-dns-dnsimple', + '-e certbot-dns-dnsmadeeasy', + '-e certbot-dns-gehirn', + '-e certbot-dns-google', + '-e certbot-dns-linode', + '-e certbot-dns-luadns', + '-e certbot-dns-nsone', + '-e certbot-dns-ovh', + '-e certbot-dns-rfc2136', + '-e certbot-dns-route53', + '-e certbot-dns-sakuracloud', + '-e certbot-nginx', + '-e certbot-postfix', + '-e letshelp-certbot', + '-e certbot-compatibility-test', +] + +def get_venv_args(): + with open(os.devnull, 'w') as fnull: + command_python2_st_code = subprocess.call( + 'command -v python2', shell=True, stdout=fnull, stderr=fnull) + if not command_python2_st_code: + return '--python python2' + + command_python27_st_code = subprocess.call( + 'command -v python2.7', shell=True, stdout=fnull, stderr=fnull) + if not command_python27_st_code: + return '--python python2.7' + + raise ValueError('Couldn\'t find python2 or python2.7 in {0}'.format(os.environ.get('PATH'))) + +def main(): + if os.name == 'nt': + raise ValueError('Certbot for Windows is not supported on Python 2.x.') + + venv_args = get_venv_args() + + _venv_common.main('venv', venv_args, REQUIREMENTS) + +if __name__ == '__main__': + main() diff --git a/tools/venv.sh b/tools/venv.sh deleted file mode 100755 index 5692f9ebf..000000000 --- a/tools/venv.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh -xe -# Developer virtualenv setup for Certbot client - -if command -v python2; then - export VENV_ARGS="--python python2" -elif command -v python2.7; then - export VENV_ARGS="--python python2.7" -else - echo "Couldn't find python2 or python2.7 in $PATH" - exit 1 -fi - -./tools/_venv_common.sh \ - -e acme[dev] \ - -e .[dev,docs] \ - -e certbot-apache \ - -e certbot-dns-cloudflare \ - -e certbot-dns-cloudxns \ - -e certbot-dns-digitalocean \ - -e certbot-dns-dnsimple \ - -e certbot-dns-dnsmadeeasy \ - -e certbot-dns-gehirn \ - -e certbot-dns-google \ - -e certbot-dns-linode \ - -e certbot-dns-luadns \ - -e certbot-dns-nsone \ - -e certbot-dns-ovh \ - -e certbot-dns-rfc2136 \ - -e certbot-dns-route53 \ - -e certbot-dns-sakuracloud \ - -e certbot-nginx \ - -e certbot-postfix \ - -e letshelp-certbot \ - -e certbot-compatibility-test diff --git a/tools/venv3.py b/tools/venv3.py new file mode 100755 index 000000000..15db9495a --- /dev/null +++ b/tools/venv3.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# Developer virtualenv setup for Certbot client + +from __future__ import absolute_import + +import os +import subprocess + +import _venv_common + +REQUIREMENTS = [ + '-e acme[dev]', + '-e .[dev,docs]', + '-e certbot-apache', + '-e certbot-dns-cloudflare', + '-e certbot-dns-cloudxns', + '-e certbot-dns-digitalocean', + '-e certbot-dns-dnsimple', + '-e certbot-dns-dnsmadeeasy', + '-e certbot-dns-gehirn', + '-e certbot-dns-google', + '-e certbot-dns-linode', + '-e certbot-dns-luadns', + '-e certbot-dns-nsone', + '-e certbot-dns-ovh', + '-e certbot-dns-rfc2136', + '-e certbot-dns-route53', + '-e certbot-dns-sakuracloud', + '-e certbot-nginx', + '-e certbot-postfix', + '-e letshelp-certbot', + '-e certbot-compatibility-test', +] + +def get_venv_args(): + with open(os.devnull, 'w') as fnull: + where_python3_st_code = subprocess.call( + 'where python3', shell=True, stdout=fnull, stderr=fnull) + command_python3_st_code = subprocess.call( + 'command -v python3', shell=True, stdout=fnull, stderr=fnull) + + if not where_python3_st_code or not command_python3_st_code: + return '--python python3' + + raise ValueError('Couldn\'t find python3 in {0}'.format(os.environ.get('PATH'))) + +def main(): + venv_args = get_venv_args() + + _venv_common.main('venv3', venv_args, REQUIREMENTS) + +if __name__ == '__main__': + main() diff --git a/tools/venv3.sh b/tools/venv3.sh deleted file mode 100755 index 07512f370..000000000 --- a/tools/venv3.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -xe -# Developer Python3 virtualenv setup for Certbot - -if command -v python3; then - export VENV_NAME="${VENV_NAME:-venv3}" - export VENV_ARGS="--python python3" -else - echo "Couldn't find python3 in $PATH" - exit 1 -fi - -./tools/_venv_common.sh \ - -e acme[dev] \ - -e .[dev,docs] \ - -e certbot-apache \ - -e certbot-dns-cloudflare \ - -e certbot-dns-cloudxns \ - -e certbot-dns-digitalocean \ - -e certbot-dns-dnsimple \ - -e certbot-dns-dnsmadeeasy \ - -e certbot-dns-gehirn \ - -e certbot-dns-google \ - -e certbot-dns-linode \ - -e certbot-dns-luadns \ - -e certbot-dns-nsone \ - -e certbot-dns-ovh \ - -e certbot-dns-rfc2136 \ - -e certbot-dns-route53 \ - -e certbot-dns-sakuracloud \ - -e certbot-nginx \ - -e certbot-postfix \ - -e letshelp-certbot \ - -e certbot-compatibility-test diff --git a/tox-win.ini b/tox-win.ini deleted file mode 100644 index fe063c264..000000000 --- a/tox-win.ini +++ /dev/null @@ -1,13 +0,0 @@ -[tox] -skipsdist = True -envlist = py{34,35,36,37}-cover - -[testenv] -deps = -e acme[dev] - -e .[dev] -commands = pytest -n auto --pyargs acme - pytest -n auto --pyargs certbot - -[testenv:cover] -commands = pytest -n auto --cov acme --pyargs acme - pytest -n auto --cov certbot --cov-append --pyargs certbot diff --git a/tox.cover.py b/tox.cover.py new file mode 100755 index 000000000..8bbce2d09 --- /dev/null +++ b/tox.cover.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +import argparse +import subprocess +import os +import sys + +DEFAULT_PACKAGES = [ + 'certbot', 'acme', 'certbot_apache', 'certbot_dns_cloudflare', 'certbot_dns_cloudxns', + 'certbot_dns_digitalocean', 'certbot_dns_dnsimple', 'certbot_dns_dnsmadeeasy', + 'certbot_dns_gehirn', 'certbot_dns_google', 'certbot_dns_linode', 'certbot_dns_luadns', + 'certbot_dns_nsone', 'certbot_dns_ovh', 'certbot_dns_rfc2136', 'certbot_dns_route53', + 'certbot_dns_sakuracloud', 'certbot_nginx', 'certbot_postfix', 'letshelp_certbot'] + +COVER_THRESHOLDS = { + 'certbot': 98, + 'acme': 100, + 'certbot_apache': 100, + 'certbot_dns_cloudflare': 98, + 'certbot_dns_cloudxns': 99, + 'certbot_dns_digitalocean': 98, + 'certbot_dns_dnsimple': 98, + 'certbot_dns_dnsmadeeasy': 99, + 'certbot_dns_gehirn': 97, + 'certbot_dns_google': 99, + 'certbot_dns_linode': 98, + 'certbot_dns_luadns': 98, + 'certbot_dns_nsone': 99, + 'certbot_dns_ovh': 97, + 'certbot_dns_rfc2136': 99, + 'certbot_dns_route53': 92, + 'certbot_dns_sakuracloud': 97, + 'certbot_nginx': 97, + 'certbot_postfix': 100, + 'letshelp_certbot': 100 +} + +SKIP_PROJECTS_ON_WINDOWS = [ + 'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot'] + +def cover(package): + threshold = COVER_THRESHOLDS.get(package) + + if not threshold: + raise ValueError('Unrecognized package: {0}'.format(package)) + + pkg_dir = package.replace('_', '-') + + if os.name == 'nt' and pkg_dir in SKIP_PROJECTS_ON_WINDOWS: + print(( + 'Info: currently {0} is not supported on Windows and will not be tested/covered.' + .format(pkg_dir))) + return + + subprocess.call([ + sys.executable, '-m', 'pytest', '--cov', pkg_dir, '--cov-append', '--cov-report=', + '--numprocesses', 'auto', '--pyargs', package]) + subprocess.call([ + sys.executable, '-m', 'coverage', 'report', '--fail-under', str(threshold), '--include', + '{0}/*'.format(pkg_dir), '--show-missing']) + +def main(): + description = """ +This script is used by tox.ini (and thus by Travis CI and AppVeyor) in order +to generate separate stats for each package. It should be removed once those +packages are moved to a separate repo. + +Option -e makes sure we fail fast and don't submit to codecov.""" + parser = argparse.ArgumentParser(description=description) + parser.add_argument('--packages', nargs='+') + + args = parser.parse_args() + + packages = args.packages or DEFAULT_PACKAGES + + # --cov-append is on, make sure stats are correct + try: + os.remove('.coverage') + except OSError: + pass + + for package in packages: + cover(package) + +if __name__ == '__main__': + main() diff --git a/tox.cover.sh b/tox.cover.sh deleted file mode 100755 index c68e757de..000000000 --- a/tox.cover.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/sh -xe - -# USAGE: ./tox.cover.sh [package] -# -# This script is used by tox.ini (and thus Travis CI) in order to -# generate separate stats for each package. It should be removed once -# those packages are moved to separate repo. -# -# -e makes sure we fail fast and don't submit to codecov - -if [ "xxx$1" = "xxx" ]; then - pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_gehirn certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_ovh certbot_dns_rfc2136 certbot_dns_route53 certbot_dns_sakuracloud certbot_nginx certbot_postfix letshelp_certbot" -else - pkgs="$@" -fi - -cover () { - if [ "$1" = "certbot" ]; then - min=98 - elif [ "$1" = "acme" ]; then - min=100 - elif [ "$1" = "certbot_apache" ]; then - min=100 - elif [ "$1" = "certbot_dns_cloudflare" ]; then - min=98 - elif [ "$1" = "certbot_dns_cloudxns" ]; then - min=99 - elif [ "$1" = "certbot_dns_digitalocean" ]; then - min=98 - elif [ "$1" = "certbot_dns_dnsimple" ]; then - min=98 - elif [ "$1" = "certbot_dns_dnsmadeeasy" ]; then - min=99 - elif [ "$1" = "certbot_dns_gehirn" ]; then - min=97 - elif [ "$1" = "certbot_dns_google" ]; then - min=99 - elif [ "$1" = "certbot_dns_linode" ]; then - min=98 - elif [ "$1" = "certbot_dns_luadns" ]; then - min=98 - elif [ "$1" = "certbot_dns_nsone" ]; then - min=99 - elif [ "$1" = "certbot_dns_ovh" ]; then - min=97 - elif [ "$1" = "certbot_dns_rfc2136" ]; then - min=99 - elif [ "$1" = "certbot_dns_route53" ]; then - min=92 - elif [ "$1" = "certbot_dns_sakuracloud" ]; then - min=97 - elif [ "$1" = "certbot_nginx" ]; then - min=97 - elif [ "$1" = "certbot_postfix" ]; then - min=100 - elif [ "$1" = "letshelp_certbot" ]; then - min=100 - else - echo "Unrecognized package: $1" - exit 1 - fi - - pkg_dir=$(echo "$1" | tr _ -) - pytest --cov "$pkg_dir" --cov-append --cov-report= --numprocesses "auto" --pyargs "$1" - coverage report --fail-under="$min" --include="$pkg_dir/*" --show-missing -} - -rm -f .coverage # --cov-append is on, make sure stats are correct -for pkg in $pkgs -do - cover $pkg -done diff --git a/tox.ini b/tox.ini index 9db06f78c..e38f1311e 100644 --- a/tox.ini +++ b/tox.ini @@ -4,16 +4,16 @@ [tox] skipsdist = true -envlist = modification,py{34,35,36},cover,lint +envlist = modification,py{34,35,36},py27-cover,lint [base] # pip installs the requested packages in editable mode -pip_install = {toxinidir}/tools/pip_install_editable.sh +pip_install = python {toxinidir}/tools/pip_install_editable.py # pip installs the requested packages in editable mode and runs unit tests on # them. Each package is installed and tested in the order they are provided # before the script moves on to the next package. All dependencies are pinned # to a specific version for increased stability for developers. -install_and_test = {toxinidir}/tools/install_and_test.sh +install_and_test = python {toxinidir}/tools/install_and_test.py dns_packages = certbot-dns-cloudflare \ certbot-dns-cloudxns \ @@ -38,7 +38,7 @@ all_packages = certbot-postfix \ letshelp-certbot install_packages = - {toxinidir}/tools/pip_install_editable.sh {[base]all_packages} + python {toxinidir}/tools/pip_install_editable.py {[base]all_packages} source_paths = acme/acme certbot @@ -64,7 +64,9 @@ source_paths = tests/lock_test.py [testenv] -passenv = TRAVIS +passenv = + TRAVIS + APPVEYOR commands = {[base]install_and_test} {[base]all_packages} python tests/lock_test.py @@ -120,11 +122,17 @@ basepython = python2.7 commands = {[base]install_packages} -[testenv:cover] +[testenv:py27-cover] basepython = python2.7 commands = {[base]install_packages} - ./tox.cover.sh + python tox.cover.py + +[testenv:py37-cover] +basepython = python3.7 +commands = + {[base]install_packages} + python tox.cover.py [testenv:lint] basepython = python2.7 @@ -133,7 +141,7 @@ basepython = python2.7 # continue, but tox return code will reflect previous error commands = {[base]install_packages} - pylint --reports=n --rcfile=.pylintrc {[base]source_paths} + python -m pylint --reports=n --rcfile=.pylintrc {[base]source_paths} [testenv:mypy] basepython = python3 @@ -157,7 +165,7 @@ commands = # allow users to run the modification check by running `tox` [testenv:modification] commands = - {toxinidir}/tests/modification-check.sh + python {toxinidir}/tests/modification-check.py [testenv:apache_compat] commands = @@ -197,7 +205,7 @@ passenv = # At the moment, this tests under Python 2.7 only, as only that version is # readily available on the Trusty Docker image. commands = - {toxinidir}/tests/modification-check.sh + python {toxinidir}/tests/modification-check.py docker build -f letsencrypt-auto-source/Dockerfile.trusty -t lea letsencrypt-auto-source docker run --rm -t -i lea whitelist_externals = -- cgit v1.2.3 From 7352727a6507ee153dff485b5423940497186663 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 8 Nov 2018 17:35:07 +0100 Subject: [URGENT] Fix the CI system (#6485) It is about the exit codes that are returned from the various scripts in tools during tox execution. Indeed, tox relies on the non-zero exit code from a given script to know that something failed during the execution. Previously, theses scripts were in bash, and a bash script returns an exit code that is the higher code returned from any of the command executed by the script. So if any command return a non-zero (in particular pylint or pytest), then the script return also non-zero. Now that these scripts are converted into python, pylint and pytest are executed via subprocess, that returns the exit code as variables. But if theses codes are not handled explicitly, the python script itself will return zero if no python exception occured. As a consequence currently, Certbot CI system is unable to detect any test error or lint error, because there is no exception in this case, only exit codes from the binaries executed. This PR fixes that, by handling correctly the exit code from the most critical scripts, install_and_test.py and tox.cover.py, but also all the scripts that I converted into Python and that could be executed in the context of a shell (via tox or directly for instance). --- tools/_venv_common.py | 20 +++++++++++++------- tools/install_and_test.py | 14 +++++++++----- tools/pip_install.py | 15 ++++++++++----- tools/pip_install_editable.py | 5 +++-- tools/venv.py | 5 +++-- tools/venv3.py | 5 +++-- 6 files changed, 41 insertions(+), 23 deletions(-) diff --git a/tools/_venv_common.py b/tools/_venv_common.py index 0c24664b3..cce88f826 100755 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -11,7 +11,7 @@ import sys def subprocess_with_print(command): print(command) - subprocess.call(command, shell=True) + return subprocess.call(command, shell=True) def get_venv_python(venv_path): python_linux = os.path.join(venv_path, 'bin/python') @@ -35,17 +35,19 @@ def main(venv_name, venv_args, args): if os.path.isdir(venv_name): os.rename(venv_name, '{0}.{1}.bak'.format(venv_name, int(time.time()))) - subprocess_with_print(' '.join([ + exit_code = 0 + + exit_code = subprocess_with_print(' '.join([ sys.executable, '-m', 'virtualenv', '--no-site-packages', '--setuptools', - venv_name, venv_args])) + venv_name, venv_args])) or exit_code python_executable = get_venv_python(venv_name) - subprocess_with_print(' '.join([ + exit_code = subprocess_with_print(' '.join([ python_executable, os.path.normpath('./letsencrypt-auto-source/pieces/pipstrap.py')])) - command = [python_executable, os.path.normpath('./tools/pip_install.py')] + command = [python_executable, os.path.normpath('./tools/pip_install.py')] or exit_code command.extend(args) - subprocess_with_print(' '.join(command)) + exit_code = subprocess_with_print(' '.join(command)) or exit_code if os.path.isdir(os.path.join(venv_name, 'bin')): # Linux/OSX specific @@ -63,5 +65,9 @@ def main(venv_name, venv_args, args): else: raise ValueError('Error, directory {0} is not a valid venv.'.format(venv_name)) + return exit_code + if __name__ == '__main__': - main(os.environ.get('VENV_NAME', 'venv'), os.environ.get('VENV_ARGS', ''), sys.argv[1:]) + sys.exit(main(os.environ.get('VENV_NAME', 'venv'), + os.environ.get('VENV_ARGS', ''), + sys.argv[1:])) diff --git a/tools/install_and_test.py b/tools/install_and_test.py index b16181aa5..149ffc776 100755 --- a/tools/install_and_test.py +++ b/tools/install_and_test.py @@ -19,7 +19,7 @@ SKIP_PROJECTS_ON_WINDOWS = [ def call_with_print(command, cwd=None): print(command) - subprocess.call(command, shell=True, cwd=cwd or os.getcwd()) + return subprocess.call(command, shell=True, cwd=cwd or os.getcwd()) def main(args): if os.environ.get('CERTBOT_NO_PIN') == '1': @@ -37,10 +37,12 @@ def main(args): else: new_args.append(arg) + exit_code = 0 + for requirement in new_args: current_command = command[:] current_command.append(requirement) - call_with_print(' '.join(current_command)) + exit_code = call_with_print(' '.join(current_command)) or exit_code pkg = re.sub(r'\[\w+\]', '', requirement) if pkg == '.': @@ -48,11 +50,13 @@ def main(args): temp_cwd = tempfile.mkdtemp() try: - call_with_print(' '.join([ + exit_code = call_with_print(' '.join([ sys.executable, '-m', 'pytest', '--numprocesses', 'auto', - '--quiet', '--pyargs', pkg.replace('-', '_')]), cwd=temp_cwd) + '--quiet', '--pyargs', pkg.replace('-', '_')]), cwd=temp_cwd) or exit_code finally: shutil.rmtree(temp_cwd) + return exit_code + if __name__ == '__main__': - main(sys.argv[1:]) + sys.exit(main(sys.argv[1:])) diff --git a/tools/pip_install.py b/tools/pip_install.py index d09997bf5..273ce5ec2 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -59,11 +59,14 @@ def merge_requirements(tools_path, test_constraints, all_constraints): def call_with_print(command, cwd=None): print(command) - subprocess.call(command, shell=True, cwd=cwd or os.getcwd()) + return subprocess.call(command, shell=True, cwd=cwd or os.getcwd()) def main(args): tools_path = find_tools_path() working_dir = tempfile.mkdtemp() + + exit_code = 0 + try: test_constraints = os.path.join(working_dir, 'test_constraints.txt') all_constraints = os.path.join(working_dir, 'all_constraints.txt') @@ -76,15 +79,17 @@ def main(args): merge_requirements(tools_path, test_constraints, all_constraints) if requirements: - call_with_print(' '.join([ + exit_code = call_with_print(' '.join([ sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints, - '--requirement', requirements])) + '--requirement', requirements])) or exit_code command = [sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints] command.extend(args) - call_with_print(' '.join(command)) + exit_code = call_with_print(' '.join(command)) or exit_code finally: shutil.rmtree(working_dir) + return exit_code + if __name__ == '__main__': - main(sys.argv[1:]) + sys.exit(main(sys.argv[1:])) diff --git a/tools/pip_install_editable.py b/tools/pip_install_editable.py index bdbdcadc5..35cc2264d 100755 --- a/tools/pip_install_editable.py +++ b/tools/pip_install_editable.py @@ -13,7 +13,8 @@ def main(args): for arg in args: new_args.append('-e') new_args.append(arg) - pip_install.main(new_args) + + return pip_install.main(new_args) if __name__ == '__main__': - main(sys.argv[1:]) + sys.exit(main(sys.argv[1:])) diff --git a/tools/venv.py b/tools/venv.py index 849c49119..2cc43251d 100755 --- a/tools/venv.py +++ b/tools/venv.py @@ -5,6 +5,7 @@ from __future__ import absolute_import import os import subprocess +import sys import _venv_common @@ -52,7 +53,7 @@ def main(): venv_args = get_venv_args() - _venv_common.main('venv', venv_args, REQUIREMENTS) + return _venv_common.main('venv', venv_args, REQUIREMENTS) if __name__ == '__main__': - main() + sys.exit(main()) diff --git a/tools/venv3.py b/tools/venv3.py index 15db9495a..1bacc9c9a 100755 --- a/tools/venv3.py +++ b/tools/venv3.py @@ -5,6 +5,7 @@ from __future__ import absolute_import import os import subprocess +import sys import _venv_common @@ -47,7 +48,7 @@ def get_venv_args(): def main(): venv_args = get_venv_args() - _venv_common.main('venv3', venv_args, REQUIREMENTS) + return _venv_common.main('venv3', venv_args, REQUIREMENTS) if __name__ == '__main__': - main() + sys.exit(main()) -- cgit v1.2.3 From ad885afdb8f51568dd247f1df5e9e1caf99ade12 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Sat, 10 Nov 2018 01:17:17 +0100 Subject: Correct venv3 detection on windows (#6490) A little typo in the _venv_common.py block the script to finish correctly once the virtual environment has been setup on Windows. This PR fixes that. --- tools/_venv_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/_venv_common.py b/tools/_venv_common.py index cce88f826..b180518f9 100755 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -55,7 +55,7 @@ def main(venv_name, venv_args, args): print('Please run the following command to activate developer environment:') print('source {0}/bin/activate'.format(venv_name)) print('-------------------------------------------------------------------') - elif os.path.isdir(os.path.join(venv_args, 'Scripts')): + elif os.path.isdir(os.path.join(venv_name, 'Scripts')): # Windows specific print('---------------------------------------------------------------------------') print('Please run one of the following commands to activate developer environment:') -- cgit v1.2.3