diff options
author | Bernhard Posselt <BernhardPosselt@users.noreply.github.com> | 2016-06-03 02:38:37 +0300 |
---|---|---|
committer | Bernhard Posselt <BernhardPosselt@users.noreply.github.com> | 2016-06-03 02:38:37 +0300 |
commit | 0e25099594025ced157ecbb81c0683f8810fafb0 (patch) | |
tree | 6c10895fd3540edbdbdf12cfb4701dce660b775c | |
parent | a6c413e6ad836e7873f296c9844d583341f4798b (diff) | |
parent | 00c355565ef73acc5163e810b92abea8d5721af0 (diff) |
Merge pull request #1 from nextcloud/typings
Typings
-rw-r--r-- | .travis.yml | 2 | ||||
-rw-r--r-- | Makefile | 7 | ||||
-rw-r--r-- | README.rst | 2 | ||||
-rw-r--r-- | nextcloud_news_updater/__main__.py | 2 | ||||
-rw-r--r-- | nextcloud_news_updater/api/api.py | 7 | ||||
-rw-r--r-- | nextcloud_news_updater/api/cli.py | 53 | ||||
-rw-r--r-- | nextcloud_news_updater/api/updater.py | 87 | ||||
-rw-r--r-- | nextcloud_news_updater/api/web.py | 47 | ||||
-rw-r--r-- | nextcloud_news_updater/common/argumentparser.py | 18 | ||||
-rw-r--r-- | nextcloud_news_updater/common/logger.py | 8 | ||||
-rw-r--r-- | nextcloud_news_updater/config.py | 27 | ||||
-rw-r--r-- | nextcloud_news_updater/container.py | 6 | ||||
-rw-r--r-- | nextcloud_news_updater/dependencyinjection/container.py | 30 | ||||
-rw-r--r-- | nextcloud_news_updater/version.py | 2 | ||||
-rw-r--r-- | setup.py | 7 |
15 files changed, 170 insertions, 135 deletions
diff --git a/.travis.yml b/.travis.yml index a3de36e..fa7de71 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ python: - "3.5" before_install: - - pip install pep8 + - pip install pep8 mypy-lang script: - make test
\ No newline at end of file @@ -33,3 +33,10 @@ test: pep8 . python3 -m nextcloud_news_updater --version python3 -m unittest + #uncomment once mypy works properly + # make typecheck + +.PHONY: typecheck +typecheck: + python3 -m mypy $(CURDIR)/nextcloud_news_updater --disallow-untyped-defs + @@ -27,7 +27,7 @@ Dependencies ------------ * **Python >=3.4** - +* **typing** (from pip) if you are running Python 3.4 Pre-Installation ---------------- diff --git a/nextcloud_news_updater/__main__.py b/nextcloud_news_updater/__main__.py index d2e79ec..2357fe8 100644 --- a/nextcloud_news_updater/__main__.py +++ b/nextcloud_news_updater/__main__.py @@ -21,7 +21,7 @@ if sys.version_info < (3, 4): exit(1) -def main(): +def main() -> None: container = Container() container.resolve(Updater).run() diff --git a/nextcloud_news_updater/api/api.py b/nextcloud_news_updater/api/api.py index 1f197f4..837583c 100644 --- a/nextcloud_news_updater/api/api.py +++ b/nextcloud_news_updater/api/api.py @@ -1,4 +1,5 @@ import json +from typing import List, Any class Feed: @@ -6,13 +7,13 @@ class Feed: Payload object for update infos """ - def __init__(self, feed_id, user_id): + def __init__(self, feed_id: int, user_id: str) -> None: self.feed_id = feed_id self.user_id = user_id class Api: - def parse_feed(self, json_string): + def parse_feed(self, json_string: str) -> List[Feed]: """ Wrapper around json.loads for better error messages """ @@ -23,6 +24,6 @@ class Api: msg = "Could not parse given JSON: %s" % json_string raise ValueError(msg) - def _parse_json(self, feed_json): + def _parse_json(self, feed_json: Any) -> List[Feed]: feed_json = feed_json['feeds'] return [Feed(info['id'], info['userId']) for info in feed_json] diff --git a/nextcloud_news_updater/api/cli.py b/nextcloud_news_updater/api/cli.py index 5bb7bd6..41307fc 100644 --- a/nextcloud_news_updater/api/cli.py +++ b/nextcloud_news_updater/api/cli.py @@ -1,4 +1,5 @@ from subprocess import check_output +from typing import List, Any from nextcloud_news_updater.api.api import Api, Feed from nextcloud_news_updater.api.updater import Updater, UpdateThread @@ -7,12 +8,12 @@ from nextcloud_news_updater.config import Config class Cli: - def run(self, commands): + def run(self, commands: List[str]) -> bytes: return check_output(commands) class CliApi(Api): - def __init__(self, config): + def __init__(self, config: Config) -> None: directory = config.url phpini = config.phpini if not directory.endswith('/'): @@ -30,57 +31,59 @@ class CliApi(Api): class CliApiV2(CliApi): - def __init__(self, config): + def __init__(self, config: Config) -> None: super().__init__(config) - def _parse_json(self, feed_json): + def _parse_json(self, feed_json: Any) -> List[Feed]: feed_json = feed_json['updater'] return [Feed(info['feedId'], info['userId']) for info in feed_json] -def create_cli_api(config): +def create_cli_api(config: Config) -> CliApi: if config.apilevel == 'v1-2': return CliApi(config) if config.apilevel == 'v2': return CliApiV2(config) +class CliUpdateThread(UpdateThread): + def __init__(self, feeds: List[Feed], logger: Logger, api: CliApi, + cli: Cli) -> None: + super().__init__(feeds, logger) + self.cli = cli + self.api = api + + def update_feed(self, feed: Feed) -> None: + command = self.api.update_feed_command + [str(feed.feed_id), + feed.user_id] + self.logger.info('Running update command: %s' % ' '.join(command)) + self.cli.run(command) + + class CliUpdater(Updater): - def __init__(self, config: Config, logger: Logger, api: CliApi, cli: Cli): + def __init__(self, config: Config, logger: Logger, api: CliApi, + cli: Cli) -> None: super().__init__(config, logger) self.cli = cli self.api = api - def before_update(self): + def before_update(self) -> None: self.logger.info('Running before update command: %s' % ' '.join(self.api.before_cleanup_command)) self.cli.run(self.api.before_cleanup_command) - def start_update_thread(self, feeds): + def start_update_thread(self, feeds: List[Feed]) -> CliUpdateThread: return CliUpdateThread(feeds, self.logger, self.api, self.cli) - def all_feeds(self): - feeds_json = self.cli.run(self.api.all_feeds_command).strip() - feeds_json = str(feeds_json, 'utf-8') + def all_feeds(self) -> List[Feed]: + feeds_json_bytes = self.cli.run(self.api.all_feeds_command).strip() + feeds_json = str(feeds_json_bytes, 'utf-8') self.logger.info('Running get all feeds command: %s' % ' '.join(self.api.all_feeds_command)) self.logger.info('Received these feeds to update: %s' % feeds_json) return self.api.parse_feed(feeds_json) - def after_update(self): + def after_update(self) -> None: self.logger.info('Running after update command: %s' % ' '.join(self.api.after_cleanup_command)) self.cli.run(self.api.after_cleanup_command) - - -class CliUpdateThread(UpdateThread): - def __init__(self, feeds, logger, api, cli): - super().__init__(feeds, logger) - self.cli = cli - self.api = api - - def update_feed(self, feed): - command = self.api.update_feed_command + [str(feed.feed_id), - feed.user_id] - self.logger.info('Running update command: %s' % ' '.join(command)) - self.cli.run(command) diff --git a/nextcloud_news_updater/api/updater.py b/nextcloud_news_updater/api/updater.py index 05bdeb1..6c19f4e 100644 --- a/nextcloud_news_updater/api/updater.py +++ b/nextcloud_news_updater/api/updater.py @@ -2,6 +2,46 @@ import sys import threading import time import traceback +from typing import List + +from nextcloud_news_updater.api.api import Feed +from nextcloud_news_updater.common.logger import Logger +from nextcloud_news_updater.config import Config + + +class UpdateThread(threading.Thread): + """ + Baseclass for the updating thread which executes the feed updates in + parallel + """ + lock = threading.Lock() + + def __init__(self, feeds: List[Feed], logger: Logger) -> None: + super().__init__() + self.feeds = feeds + self.logger = logger + + def run(self) -> None: + while True: + with UpdateThread.lock: + if len(self.feeds) > 0: + feed = self.feeds.pop() + else: + return + try: + self.logger.info('Updating feed with id %s and user %s' % + (feed.feed_id, feed.user_id)) + self.update_feed(feed) + except Exception as e: + self.logger.error(str(e)) + traceback.print_exc(file=sys.stderr) + + def update_feed(self, feed: Feed) -> None: + """ + Updates a single feed + feed: the feed object containing the feed_id and user_id + """ + raise NotImplementedError class Updater: @@ -10,11 +50,11 @@ class Updater: threading and the general workflow """ - def __init__(self, config, logger): + def __init__(self, config: Config, logger: Logger) -> None: self.logger = logger self.config = config - def run(self): + def run(self) -> None: single_run = self.config.mode == 'singlerun' if single_run: self.logger.info('Running update once with %d threads' % @@ -58,49 +98,14 @@ class Updater: else: time.sleep(30) - def before_update(self): + def before_update(self) -> None: raise NotImplementedError - def start_update_thread(self, feeds): + def start_update_thread(self, feeds: List[Feed]) -> UpdateThread: raise NotImplementedError - def all_feeds(self): + def all_feeds(self) -> List[Feed]: raise NotImplementedError - def after_update(self): - raise NotImplementedError - - -class UpdateThread(threading.Thread): - """ - Baseclass for the updating thread which executes the feed updates in - parallel - """ - lock = threading.Lock() - - def __init__(self, feeds, logger): - super().__init__() - self.feeds = feeds - self.logger = logger - - def run(self): - while True: - with UpdateThread.lock: - if len(self.feeds) > 0: - feed = self.feeds.pop() - else: - return - try: - self.logger.info('Updating feed with id %s and user %s' % - (feed.feed_id, feed.user_id)) - self.update_feed(feed) - except Exception as e: - self.logger.error(e) - traceback.print_exc(file=sys.stderr) - - def update_feed(self, feed): - """ - Updates a single feed - feed: the feed object containing the feed_id and user_id - """ + def after_update(self) -> None: raise NotImplementedError diff --git a/nextcloud_news_updater/api/web.py b/nextcloud_news_updater/api/web.py index 52ec114..5516a58 100644 --- a/nextcloud_news_updater/api/web.py +++ b/nextcloud_news_updater/api/web.py @@ -1,7 +1,8 @@ import base64 -import urllib.parse -import urllib.request +from urllib.parse import urlencode +from urllib.request import Request, urlopen from collections import OrderedDict +from typing import List, Tuple, Any from nextcloud_news_updater.api.api import Api, Feed from nextcloud_news_updater.api.updater import Updater, UpdateThread @@ -10,7 +11,7 @@ from nextcloud_news_updater.config import Config class WebApi(Api): - def __init__(self, config): + def __init__(self, config: Config) -> None: base_url = config.url base_url = self._generify_base_url(base_url) self.base_url = '%sindex.php/apps/news/api/v1-2' % base_url @@ -19,14 +20,14 @@ class WebApi(Api): self.all_feeds_url = '%s/feeds/all' % self.base_url self.update_url = '%s/feeds/update' % self.base_url - def _generify_base_url(self, url): + def _generify_base_url(self, url: str) -> str: if not url.endswith('/'): url += '/' return url class WebApiV2(WebApi): - def __init__(self, config): + def __init__(self, config: Config) -> None: super().__init__(config) base_url = self._generify_base_url(config.url) self.base_url = '%sindex.php/apps/news/api/v2' % base_url @@ -35,12 +36,12 @@ class WebApiV2(WebApi): self.all_feeds_url = '%s/updater/all-feeds' % self.base_url self.update_url = '%s/updater/update-feed' % self.base_url - def _parse_json(self, feed_json): + def _parse_json(self, feed_json: Any) -> List[Feed]: feed_json = feed_json['updater'] return [Feed(info['feedId'], info['userId']) for info in feed_json] -def create_web_api(config): +def create_web_api(config: Config) -> WebApi: if config.apilevel == 'v1-2': return WebApi(config) if config.apilevel == 'v2': @@ -48,62 +49,64 @@ def create_web_api(config): class HttpClient: - def get(self, url, auth, timeout=5 * 60): + def get(self, url: str, auth: Tuple[str, str], + timeout: int = 5 * 60) -> str: """ Small wrapper for getting rid of the requests library """ - auth = bytes(auth[0] + ':' + auth[1], 'utf-8') - auth_header = 'Basic ' + base64.b64encode(auth).decode('utf-8') - req = urllib.request.Request(url) + basic_auth = bytes(':'.join(auth), 'utf-8') + auth_header = 'Basic ' + base64.b64encode(basic_auth).decode('utf-8') + req = Request(url) req.add_header('Authorization', auth_header) - response = urllib.request.urlopen(req, timeout=timeout) + response = urlopen(req, timeout=timeout) return response.read().decode('utf8') class WebUpdater(Updater): def __init__(self, config: Config, logger: Logger, api: WebApi, - client: HttpClient): + client: HttpClient) -> None: super().__init__(config, logger) self.client = client self.api = api self.auth = (config.user, config.password) - def before_update(self): + def before_update(self) -> None: self.logger.info( 'Calling before update url: %s' % self.api.before_cleanup_url) self.client.get(self.api.before_cleanup_url, self.auth) - def start_update_thread(self, feeds): + def start_update_thread(self, feeds: List[Feed]) -> UpdateThread: return WebUpdateThread(feeds, self.config, self.logger, self.api, self.client) - def all_feeds(self): + def all_feeds(self) -> List[Feed]: feeds_json = self.client.get(self.api.all_feeds_url, self.auth) self.logger.info('Received these feeds to update: %s' % feeds_json) return self.api.parse_feed(feeds_json) - def after_update(self): + def after_update(self) -> None: self.logger.info( 'Calling after update url: %s' % self.api.after_cleanup_url) self.client.get(self.api.after_cleanup_url, self.auth) class WebUpdateThread(UpdateThread): - def __init__(self, feeds, config, logger, api, client): + def __init__(self, feeds: List[Feed], config: Config, logger: Logger, + api: WebApi, client: HttpClient) -> None: super().__init__(feeds, logger) self.client = client self.api = api self.auth = (config.user, config.password) self.config = config - def update_feed(self, feed): + def update_feed(self, feed: Feed) -> None: # make sure that the order is always defined for making it easier # to test and reason about, normal dicts are not ordered data = OrderedDict([ ('userId', feed.user_id), - ('feedId', feed.feed_id), + ('feedId', str(feed.feed_id)), ]) - data = urllib.parse.urlencode(data) - url = '%s?%s' % (self.api.update_url, data) + url_data = urlencode(data) + url = '%s?%s' % (self.api.update_url, url_data) self.logger.info('Calling update url: %s' % url) self.client.get(url, self.auth, self.config.timeout) diff --git a/nextcloud_news_updater/common/argumentparser.py b/nextcloud_news_updater/common/argumentparser.py index 59fdc85..1de35ff 100644 --- a/nextcloud_news_updater/common/argumentparser.py +++ b/nextcloud_news_updater/common/argumentparser.py @@ -1,10 +1,11 @@ import argparse +from typing import Any from nextcloud_news_updater.version import get_version class ArgumentParser: - def __init__(self): + def __init__(self) -> None: self.parser = argparse.ArgumentParser() self.parser.add_argument('--threads', '-t', help='How many feeds should be fetched in ' @@ -41,13 +42,14 @@ class ArgumentParser: 'updater. If omitted, the default one ' 'will be used') self.parser.add_argument('--user', '-u', - help='Admin username to log into ownCloud. ' + help='Admin username to log into Nextcloud. ' 'Must be specified on the command line ' 'or in the config file if the updater ' 'should update over HTTP') self.parser.add_argument('--password', '-p', - help='Admin password to log into ownCloud if ' - 'the updater should update over HTTP') + help='Admin password to log into Nextcloud ' + 'if the updater should update over HTTP' + ) self.parser.add_argument('--version', '-v', action='version', version=get_version(), help='Prints the updater\'s version') @@ -59,8 +61,8 @@ class ArgumentParser: choices=['endless', 'singlerun']) self.parser.add_argument('url', help='The URL or absolute path to the ' - 'directory where owncloud is installed. ' - 'Must be specified on the command line ' + 'directory where Nextcloud is installed.' + ' Must be specified on the command line ' 'or in the config file. If the URL ' 'starts with http:// or https://, a ' 'user and password are required. ' @@ -69,8 +71,8 @@ class ArgumentParser: '8.1.0', nargs='?') - def parse(self): + def parse(self) -> Any: return self.parser.parse_args() - def print_help(self, file): + def print_help(self, file: Any) -> None: self.parser.print_help(file) diff --git a/nextcloud_news_updater/common/logger.py b/nextcloud_news_updater/common/logger.py index 6c8a39a..42ed85f 100644 --- a/nextcloud_news_updater/common/logger.py +++ b/nextcloud_news_updater/common/logger.py @@ -4,17 +4,17 @@ from nextcloud_news_updater.config import Config class Logger: - def __init__(self, config: Config): + def __init__(self, config: Config) -> None: log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' logging.basicConfig(format=log_format) - self.logger = logging.getLogger('ownCloud News Updater') + self.logger = logging.getLogger('Nextcloud News Updater') if config.loglevel == 'info': self.logger.setLevel(logging.INFO) else: self.logger.setLevel(logging.ERROR) - def info(self, message): + def info(self, message: str) -> None: self.logger.info(message) - def error(self, message): + def error(self, message: str) -> None: self.logger.error(message) diff --git a/nextcloud_news_updater/config.py b/nextcloud_news_updater/config.py index 800bb03..972a39b 100644 --- a/nextcloud_news_updater/config.py +++ b/nextcloud_news_updater/config.py @@ -1,5 +1,7 @@ import configparser import os +from typing import List, Union, Any +from typing import Optional class InvalidConfigException(Exception): @@ -36,7 +38,7 @@ class Config: 'interval': Types.integer, } - def __init__(self): + def __init__(self) -> None: self.loglevel = 'error' self.interval = 15 * 60 self.timeout = 5 * 60 @@ -44,18 +46,18 @@ class Config: self.threads = 10 self.mode = 'endless' self.password = '' - self.user = None - self.url = None - self.phpini = None + self.user = None # type: Optional[str] + self.url = None # type: Optional[str] + self.phpini = None # type: Optional[str] - def is_web(self): - return self.url and (self.url.startswith('http://') or - self.url.startswith('https://')) + def is_web(self) -> bool: + return self.url is not None and (self.url.startswith('http://') or + self.url.startswith('https://')) class ConfigValidator: - def validate(self, config): - result = [] + def validate(self, config: Config) -> List[str]: + result = [] # type: List[str] if not config.url: return ['No url given'] @@ -80,7 +82,7 @@ class ConfigValidator: class ConfigParser: - def parse_file(self, path): + def parse_file(self, path: str) -> Config: parser = configparser.ConfigParser() successfully_parsed = parser.read(path) if len(successfully_parsed) <= 0: @@ -101,7 +103,8 @@ class ConfigParser: return config - def _parse_ini_value(self, type_enum, contents, key): + def _parse_ini_value(self, type_enum: int, contents: Any, key: str) -> \ + Union[str, int, bool]: if type_enum == Types.integer: return int(contents.get(key)) elif type_enum == Types.boolean: @@ -110,7 +113,7 @@ class ConfigParser: return contents.get(key) -def merge_configs(args, config): +def merge_configs(args: Any, config: Config) -> None: """ Merges values from argparse and configparser. Values from argparse will always override values from the config. Resulting values are set on the diff --git a/nextcloud_news_updater/container.py b/nextcloud_news_updater/container.py index 745b68e..e16be1f 100644 --- a/nextcloud_news_updater/container.py +++ b/nextcloud_news_updater/container.py @@ -13,20 +13,20 @@ from nextcloud_news_updater.dependencyinjection.container import \ class Container(BaseContainer): - def __init__(self): + def __init__(self) -> None: super().__init__() self.register(CliApi, lambda c: create_cli_api(c.resolve(Config))) self.register(WebApi, lambda c: create_web_api(c.resolve(Config))) self.register(Updater, self._create_updater) self.register(Config, self._create_config) - def _create_updater(self, container): + def _create_updater(self, container: BaseContainer) -> Updater: if container.resolve(Config).is_web(): return container.resolve(WebUpdater) else: return container.resolve(CliUpdater) - def _create_config(self, container): + def _create_config(self, container: BaseContainer) -> Config: parser = container.resolve(ArgumentParser) args = parser.parse() if args.config: diff --git a/nextcloud_news_updater/dependencyinjection/container.py b/nextcloud_news_updater/dependencyinjection/container.py index 01edc5c..858a4be 100644 --- a/nextcloud_news_updater/dependencyinjection/container.py +++ b/nextcloud_news_updater/dependencyinjection/container.py @@ -1,3 +1,7 @@ +from typing import Callable, Any +from typing import Dict + + class ResolveException(Exception): pass @@ -7,11 +11,11 @@ class Factory: Wrapper for non shared factories """ - def __init__(self, factory): + def __init__(self, factory: Callable[['Container'], Any]) -> None: self.factory = factory - def __call__(self, *args, **kwargs): - return self.factory(*args, **kwargs) + def __call__(self, container: 'Container') -> Any: + return self.factory(container) class SingletonFactory(Factory): @@ -19,7 +23,7 @@ class SingletonFactory(Factory): Wrapper for shared factories """ - def __init__(self, factory): + def __init__(self, factory: Callable[['Container'], Any]) -> None: super().__init__(factory) @@ -28,11 +32,12 @@ class Container: Simple container for Dependency Injection """ - def __init__(self): - self._singletons = {} - self._factories = {} + def __init__(self) -> None: + self._singletons = {} # type: Dict[Any, Any] + self._factories = {} # type: Dict[Any, Factory] - def register(self, key, factory, shared=True): + def register(self, key: Any, factory: Callable[['Container'], Any], + shared: bool = True) -> None: """ Registers a factory function for creating the class :argument key name under which the dependencies will be @@ -49,7 +54,7 @@ class Container: else: self._factories[key] = Factory(factory) - def resolve(self, key): + def resolve(self, key: Any) -> Any: """ Fetches an instance or creates one using the registered factory method :argument key the key to look up @@ -75,7 +80,7 @@ class Container: else: return self._singletons[key] - def alias(self, source, target): + def alias(self, source: type, target: Any) -> Any: """ Point a key to another key :argument source the key to resolve when the target is requested @@ -83,7 +88,7 @@ class Container: """ self.register(target, lambda c: c.resolve(source)) - def _resolve_class(self, clazz): + def _resolve_class(self, clazz: Any) -> object: """ Constructs an instance based on the function annotations on an object's __init__ method @@ -94,5 +99,6 @@ class Container: if hasattr(clazz.__init__, '__annotations__'): annotations = clazz.__init__.__annotations__ for name, type_hint in annotations.items(): - arguments[name] = self.resolve(type_hint) + if name != 'return': + arguments[name] = self.resolve(type_hint) return clazz(**arguments) diff --git a/nextcloud_news_updater/version.py b/nextcloud_news_updater/version.py index f1c6829..1ec38a3 100644 --- a/nextcloud_news_updater/version.py +++ b/nextcloud_news_updater/version.py @@ -1,7 +1,7 @@ from os.path import dirname, realpath, join -def get_version(): +def get_version() -> str: directory = dirname(realpath(__file__)) version_file = join(directory, 'version.txt') with open(version_file, 'r') as infile: @@ -6,6 +6,11 @@ if version_info < (3, 4): print('Error: Python 3.4 required but found %s' % python_version()) exit(1) +if version_info < (3, 5): + install_requires = ['typing'] +else: + install_requires = [] + with open('README.rst', 'r') as infile: long_description = infile.read() @@ -24,8 +29,8 @@ setup( packages=find_packages(), include_package_data=True, license='GPL', - install_requires=[], keywords=['nextcloud', 'news', 'updater', 'RSS', 'Atom', 'feed', 'reader'], + install_requires=install_requires, classifiers=[ 'Intended Audience :: System Administrators', 'Environment :: Console', |