Better error messages for JSON errors, fix install warning8.3.1

9 files changed, 231 insertions, 209 deletions

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 191ba99..b652d93 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -2,6 +2,15 @@
 
 Changelog
 ---------
+
+8.3.1
++++++
+
+**Bugfixes**
+
+- Get rid of the requirements.txt missing warning when install the lib
+- If a JSON parsing error happens, create a more helpful error message
+
 8.3.0
 +++++
 
diff --git a/owncloud_news_updater/application.py b/owncloud_news_updater/application.py
index 13f6652..04ac79a 100755
--- a/owncloud_news_updater/application.py
+++ b/owncloud_news_updater/application.py
@@ -4,14 +4,16 @@ Updater script for the news app which allows multiple feeds to be updated at
 once to speed up the update process. Built in cron has to be disabled in the
 news config, see the README.rst file in the top directory for more information.
 """
-import os
-import sys
 import argparse
 import configparser
-from owncloud_news_updater.updater import WebUpdater, ConsoleUpdater
-from owncloud_news_updater.version import get_version
+import os
+import sys
 from platform import python_version
 
+from owncloud_news_updater.updaters.cli import ConsoleUpdater
+from owncloud_news_updater.updaters.web import WebUpdater
+from owncloud_news_updater.version import get_version
+
 __author__ = 'Bernhard Posselt'
 __copyright__ = 'Copyright 2012-2016, Bernhard Posselt'
 __license__ = 'GPL3+'
diff --git a/owncloud_news_updater/updater.py b/owncloud_news_updater/updater.py
deleted file mode 100644
index a87b22f..0000000
--- a/owncloud_news_updater/updater.py
+++ /dev/null
@@ -1,204 +0,0 @@
-#!/usr/bin/env python3
-import sys
-import traceback
-import json
-import threading
-import time
-import logging
-import urllib.parse
-from subprocess import check_output
-
-from owncloud_news_updater.http import http_get
-
-
-class Updater:
-    def __init__(self, thread_num, interval, run_once, log_level):
-        self.thread_num = thread_num
-        self.run_once = run_once
-        self.interval = interval
-        # logging
-        format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
-        logging.basicConfig(format=format)
-        self.logger = logging.getLogger('ownCloud News Updater')
-        if log_level == 'info':
-            self.logger.setLevel(logging.INFO)
-        else:
-            self.logger.setLevel(logging.ERROR)
-
-    def run(self):
-        if self.run_once:
-            self.logger.info('Running update once with %d threads' %
-                             self.thread_num)
-        else:
-            self.logger.info(('Running update in an interval of %d seconds '
-                              'using %d threads') % (self.interval,
-                                                     self.thread_num))
-        while True:
-            self.start_time = time.time()  # reset clock
-            try:
-                self.before_update()
-                feeds = self.all_feeds()
-
-                threads = []
-                for num in range(0, self.thread_num):
-                    thread = self.start_update_thread(feeds)
-                    thread.start()
-                    threads.append(thread)
-                for thread in threads:
-                    thread.join()
-
-                self.after_update()
-
-                if self.run_once:
-                    return
-                # wait until the interval finished to run again and subtract
-                # the update run time from the interval
-                update_duration_seconds = int((time.time() - self.start_time))
-                timeout = self.interval - update_duration_seconds
-                if timeout > 0:
-                    self.logger.info(('Finished updating in %d seconds, '
-                                      'next update in %d seconds') %
-                                     (update_duration_seconds, timeout))
-                    time.sleep(timeout)
-            except (Exception) as e:
-                self.logger.error('%s: Trying again in 30 seconds' % e)
-                traceback.print_exc(file=sys.stderr)
-                time.sleep(30)
-
-    def before_update(self):
-        raise NotImplementedError
-
-    def start_update_thread(self, feeds):
-        raise NotImplementedError
-
-    def all_feeds(self):
-        raise NotImplementedError
-
-    def after_update(self):
-        raise NotImplementedError
-
-
-class UpdateThread(threading.Thread):
-    lock = threading.Lock()
-
-    def __init__(self, feeds, logger):
-        super().__init__()
-        self.feeds = feeds
-        self.logger = logger
-
-    def run(self):
-        while True:
-            with WebUpdateThread.lock:
-                if len(self.feeds) > 0:
-                    feed = self.feeds.pop()
-                else:
-                    return
-            try:
-                self.logger.info('Updating feed with id %s and user %s' %
-                                 (feed['id'], feed['userId']))
-                self.update_feed(feed)
-            except (Exception) as e:
-                self.logger.error(e)
-                traceback.print_exc(file=sys.stderr)
-
-    def update_feed(self, feed):
-        raise NotImplementedError
-
-
-class WebUpdater(Updater):
-    def __init__(self, base_url, thread_num, interval, run_once,
-                 user, password, timeout, log_level):
-        super().__init__(thread_num, interval, run_once, log_level)
-        self.base_url = base_url
-        self.auth = (user, password)
-        self.timeout = timeout
-
-        if self.base_url[-1] != '/':
-            self.base_url += '/'
-        self.base_url += 'index.php/apps/news/api/v1-2'
-
-        self.before_cleanup_url = '%s/cleanup/before-update' % self.base_url
-        self.after_cleanup_url = '%s/cleanup/after-update' % self.base_url
-        self.all_feeds_url = '%s/feeds/all' % self.base_url
-        self.update_url = '%s/feeds/update' % self.base_url
-
-    def before_update(self):
-        self.logger.info(
-            'Calling before update url:  %s' % self.before_cleanup_url)
-        before = http_get(self.before_cleanup_url, auth=self.auth)
-
-    def start_update_thread(self, feeds):
-        return WebUpdateThread(feeds, self.logger, self.update_url, self.auth,
-                               self.timeout)
-
-    def all_feeds(self):
-        feeds_json = http_get(self.all_feeds_url, auth=self.auth)
-        self.logger.info('Received these feeds to update: %s' % feeds_json)
-        return json.loads(feeds_json)['feeds']
-
-    def after_update(self):
-        self.logger.info(
-            'Calling after update url:  %s' % self.after_cleanup_url)
-        after = http_get(self.after_cleanup_url, auth=self.auth)
-
-
-class WebUpdateThread(UpdateThread):
-    def __init__(self, feeds, logger, update_url, auth, timeout):
-        super().__init__(feeds, logger)
-        self.update_url = update_url
-        self.auth = auth
-        self.timeout = timeout
-
-    def update_feed(self, feed):
-        # rewrite parameters, a feeds id is mapped to feedId
-        feed['feedId'] = feed['id']
-        del feed['id']
-
-        # turn the pyton dict into url parameters
-        data = urllib.parse.urlencode(feed)
-        url = '%s?%s' % (self.update_url, data)
-        request = http_get(url, auth=self.auth, timeout=self.timeout)
-
-
-class ConsoleUpdater(Updater):
-    def __init__(self, directory, thread_num, interval, run_once, log_level):
-        super().__init__(thread_num, interval, run_once, log_level)
-        self.directory = directory.rstrip('/')
-        base_command = ['php', '-f', self.directory + '/occ']
-        self.before_cleanup_command = base_command + [
-            'news:updater:before-update']
-        self.all_feeds_command = base_command + ['news:updater:all-feeds']
-        self.update_feed_command = base_command + ['news:updater:update-feed']
-        self.after_cleanup_command = base_command + [
-            'news:updater:after-update']
-
-    def before_update(self):
-        self.logger.info('Running before update command %s' %
-                         ' '.join(self.before_cleanup_command))
-        check_output(self.before_cleanup_command)
-
-    def start_update_thread(self, feeds):
-        return ConsoleUpdateThread(feeds, self.logger,
-                                   self.update_feed_command)
-
-    def all_feeds(self):
-        feeds_json = check_output(self.all_feeds_command).strip()
-        feeds_json = str(feeds_json, 'utf-8')
-        self.logger.info('Received these feeds to update: %s' % feeds_json)
-        return json.loads(feeds_json)['feeds']
-
-    def after_update(self):
-        self.logger.info('Running after update command %s' %
-                         ' '.join(self.after_cleanup_command))
-        check_output(self.before_cleanup_command)
-
-
-class ConsoleUpdateThread(UpdateThread):
-    def __init__(self, feeds, logger, update_base_command):
-        super().__init__(feeds, logger)
-        self.update_base_command = update_base_command
-
-    def update_feed(self, feed):
-        command = self.update_base_command + [str(feed['id']), feed['userId']]
-        self.logger.info('Running update command %s' % ' '.join(command))
-        check_output(command)
diff --git a/owncloud_news_updater/updaters/__init__.py b/owncloud_news_updater/updaters/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/owncloud_news_updater/updaters/__init__.py
diff --git a/owncloud_news_updater/updaters/cli.py b/owncloud_news_updater/updaters/cli.py
new file mode 100644
index 0000000..106670e
--- /dev/null
+++ b/owncloud_news_updater/updaters/cli.py
@@ -0,0 +1,48 @@
+from subprocess import check_output
+
+from owncloud_news_updater.updaters.facades import parse_json
+from owncloud_news_updater.updaters.updater import Updater, UpdateThread
+
+
+class ConsoleUpdater(Updater):
+    def __init__(self, directory, thread_num, interval, run_once, log_level):
+        super().__init__(thread_num, interval, run_once, log_level)
+        self.directory = directory.rstrip('/')
+        base_command = ['php', '-f', self.directory + '/occ']
+        self.before_cleanup_command = base_command + [
+            'news:updater:before-update']
+        self.all_feeds_command = base_command + ['news:updater:all-feeds']
+        self.update_feed_command = base_command + ['news:updater:update-feed']
+        self.after_cleanup_command = base_command + [
+            'news:updater:after-update']
+
+    def before_update(self):
+        self.logger.info('Running before update command %s' %
+                         ' '.join(self.before_cleanup_command))
+        check_output(self.before_cleanup_command)
+
+    def start_update_thread(self, feeds):
+        return ConsoleUpdateThread(feeds, self.logger,
+                                   self.update_feed_command)
+
+    def all_feeds(self):
+        feeds_json = check_output(self.all_feeds_command).strip()
+        feeds_json = str(feeds_json, 'utf-8')
+        self.logger.info('Received these feeds to update: %s' % feeds_json)
+        return parse_json(feeds_json)['feeds']
+
+    def after_update(self):
+        self.logger.info('Running after update command %s' %
+                         ' '.join(self.after_cleanup_command))
+        check_output(self.before_cleanup_command)
+
+
+class ConsoleUpdateThread(UpdateThread):
+    def __init__(self, feeds, logger, update_base_command):
+        super().__init__(feeds, logger)
+        self.update_base_command = update_base_command
+
+    def update_feed(self, feed):
+        command = self.update_base_command + [str(feed['id']), feed['userId']]
+        self.logger.info('Running update command %s' % ' '.join(command))
+        check_output(command)
diff --git a/owncloud_news_updater/http.py b/owncloud_news_updater/updaters/facades.py
index 8570c7b..9f224e8 100644
--- a/owncloud_news_updater/http.py
+++ b/owncloud_news_updater/updaters/facades.py
@@ -1,5 +1,6 @@
 import urllib.request
 import base64
+import json
 
 
 def create_basic_auth(user, password):
@@ -16,3 +17,11 @@ def http_get(url, auth, timeout=5 * 60):
     req.add_header('Authorization', auth_header)
     response = urllib.request.urlopen(req, timeout=timeout)
     return response.read().decode('utf8')
+
+
+def parse_json(json_string):
+    try:
+        return json.loads(json_string)
+    except ValueError:
+        msg = "Could not parse given JSON: %s" % json_string
+        raise ValueError(msg)
diff --git a/owncloud_news_updater/updaters/updater.py b/owncloud_news_updater/updaters/updater.py
new file mode 100644
index 0000000..6022db8
--- /dev/null
+++ b/owncloud_news_updater/updaters/updater.py
@@ -0,0 +1,99 @@
+import sys
+import traceback
+import threading
+import time
+import logging
+
+
+class Updater:
+    def __init__(self, thread_num, interval, run_once, log_level):
+        self.thread_num = thread_num
+        self.run_once = run_once
+        self.interval = interval
+        # logging
+        format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        logging.basicConfig(format=format)
+        self.logger = logging.getLogger('ownCloud News Updater')
+        if log_level == 'info':
+            self.logger.setLevel(logging.INFO)
+        else:
+            self.logger.setLevel(logging.ERROR)
+
+    def run(self):
+        if self.run_once:
+            self.logger.info('Running update once with %d threads' %
+                             self.thread_num)
+        else:
+            self.logger.info(('Running update in an interval of %d seconds '
+                              'using %d threads') % (self.interval,
+                                                     self.thread_num))
+        while True:
+            self.start_time = time.time()  # reset clock
+            try:
+                self.before_update()
+                feeds = self.all_feeds()
+
+                threads = []
+                for num in range(0, self.thread_num):
+                    thread = self.start_update_thread(feeds)
+                    thread.start()
+                    threads.append(thread)
+                for thread in threads:
+                    thread.join()
+
+                self.after_update()
+
+                if self.run_once:
+                    return
+                # wait until the interval finished to run again and subtract
+                # the update run time from the interval
+                update_duration_seconds = int((time.time() - self.start_time))
+                timeout = self.interval - update_duration_seconds
+                if timeout > 0:
+                    self.logger.info(('Finished updating in %d seconds, '
+                                      'next update in %d seconds') %
+                                     (update_duration_seconds, timeout))
+                    time.sleep(timeout)
+            except (Exception) as e:
+                self.logger.error('%s: Trying again in 30 seconds' % e)
+                traceback.print_exc(file=sys.stderr)
+                time.sleep(30)
+
+    def before_update(self):
+        raise NotImplementedError
+
+    def start_update_thread(self, feeds):
+        raise NotImplementedError
+
+    def all_feeds(self):
+        raise NotImplementedError
+
+    def after_update(self):
+        raise NotImplementedError
+
+
+class UpdateThread(threading.Thread):
+    lock = threading.Lock()
+
+    def __init__(self, feeds, logger):
+        super().__init__()
+        self.feeds = feeds
+        self.logger = logger
+
+    def run(self):
+        while True:
+            with UpdateThread.lock:
+                if len(self.feeds) > 0:
+                    feed = self.feeds.pop()
+                else:
+                    return
+            try:
+                self.logger.info('Updating feed with id %s and user %s' %
+                                 (feed['id'], feed['userId']))
+                self.update_feed(feed)
+            except (Exception) as e:
+                self.logger.error(e)
+                traceback.print_exc(file=sys.stderr)
+
+    def update_feed(self, feed):
+        raise NotImplementedError
diff --git a/owncloud_news_updater/updaters/web.py b/owncloud_news_updater/updaters/web.py
new file mode 100644
index 0000000..2bd5ad6
--- /dev/null
+++ b/owncloud_news_updater/updaters/web.py
@@ -0,0 +1,59 @@
+import urllib.parse
+
+from owncloud_news_updater.updaters.facades import http_get, parse_json
+from owncloud_news_updater.updaters.updater import Updater, UpdateThread
+
+
+class WebUpdater(Updater):
+    def __init__(self, base_url, thread_num, interval, run_once,
+                 user, password, timeout, log_level):
+        super().__init__(thread_num, interval, run_once, log_level)
+        self.base_url = base_url
+        self.auth = (user, password)
+        self.timeout = timeout
+
+        if self.base_url[-1] != '/':
+            self.base_url += '/'
+        self.base_url += 'index.php/apps/news/api/v1-2'
+
+        self.before_cleanup_url = '%s/cleanup/before-update' % self.base_url
+        self.after_cleanup_url = '%s/cleanup/after-update' % self.base_url
+        self.all_feeds_url = '%s/feeds/all' % self.base_url
+        self.update_url = '%s/feeds/update' % self.base_url
+
+    def before_update(self):
+        self.logger.info(
+            'Calling before update url:  %s' % self.before_cleanup_url)
+        http_get(self.before_cleanup_url, auth=self.auth)
+
+    def start_update_thread(self, feeds):
+        return WebUpdateThread(feeds, self.logger, self.update_url, self.auth,
+                               self.timeout)
+
+    def all_feeds(self):
+        feeds_json = http_get(self.all_feeds_url, auth=self.auth)
+        self.logger.info('Received these feeds to update: %s' % feeds_json)
+        return parse_json(feeds_json)['feeds']
+
+    def after_update(self):
+        self.logger.info(
+            'Calling after update url:  %s' % self.after_cleanup_url)
+        http_get(self.after_cleanup_url, auth=self.auth)
+
+
+class WebUpdateThread(UpdateThread):
+    def __init__(self, feeds, logger, update_url, auth, timeout):
+        super().__init__(feeds, logger)
+        self.update_url = update_url
+        self.auth = auth
+        self.timeout = timeout
+
+    def update_feed(self, feed):
+        # rewrite parameters, a feeds id is mapped to feedId
+        feed['feedId'] = feed['id']
+        del feed['id']
+
+        # turn the pyton dict into url parameters
+        data = urllib.parse.urlencode(feed)
+        url = '%s?%s' % (self.update_url, data)
+        http_get(url, auth=self.auth, timeout=self.timeout)
diff --git a/owncloud_news_updater/version.txt b/owncloud_news_updater/version.txt
index 9f4a0fb..905c243 100644
--- a/owncloud_news_updater/version.txt
+++ b/owncloud_news_updater/version.txt
@@ -1 +1 @@
-8.3.0
\ No newline at end of file
+8.3.1
\ No newline at end of file