1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
|
/*
* Android ownCloud News
*
* @author David Luhmer
* @copyright 2013 David Luhmer david-dev@live.de
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
* License as published by the Free Software Foundation; either
* version 3 of the License, or any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU AFFERO GENERAL PUBLIC LICENSE for more details.
*
* You should have received a copy of the GNU Affero General Public
* License along with this library. If not, see <http://www.gnu.org/licenses/>.
*
*/
package de.luhmer.owncloudnewsreader.reader.nextcloud;
import android.util.Log;
import com.google.gson.JsonObject;
import java.util.Date;
import java.util.List;
import java.util.UUID;
import de.luhmer.owncloudnewsreader.database.model.RssItem;
import de.luhmer.owncloudnewsreader.helper.ImageHandler;
class InsertRssItemIntoDatabase {
private final static String TAG = InsertRssItemIntoDatabase.class.getCanonicalName();
static RssItem parseItem(JsonObject e) {
Date pubDate = new Date(e.get("pubDate").getAsLong() * 1000);
String content = e.get("body").getAsString();
/*
// URL Decoding content (some pages provide url decoded content - such as showrss.info
try {
// Try URL decoding
content = URLDecoder.decode(content, "UTF-8");
} catch (UnsupportedEncodingException e1) {
e1.printStackTrace();
}
*/
//String url = e.get("url").getAsString();
String url = getStringOrDefault("url", "about:blank", e);
String guid = e.get("guid").getAsString();
String enclosureLink = getStringOrEmpty("enclosureLink", e);
String enclosureMime = getStringOrEmpty("enclosureMime", e);
String mediaDescription = getStringOrEmpty("mediaDescription", e);
Boolean rtl = getBooleanOrDefault("rtl", false, e);
if(enclosureLink.trim().equals("") && url.matches("^https?://(www.)?youtube.com/.*")) {
enclosureLink = url;
enclosureMime = "youtube";
}
RssItem rssItem = new RssItem();
rssItem.setId(e.get("id").getAsLong());
rssItem.setFeedId(e.get("feedId").getAsLong());
rssItem.setGuid(guid); // non-null
rssItem.setGuidHash(e.get("guidHash").getAsString()); // non-null
rssItem.setFingerprint(getStringOrDefault("fingerprint", "", e));
rssItem.setLastModified(new Date(Long.parseLong(getStringOrDefault("lastModified", "0", e))));
rssItem.setRead(!e.get("unread").getAsBoolean());
rssItem.setRead_temp(rssItem.getRead());
rssItem.setStarred(e.get("starred").getAsBoolean());
rssItem.setStarred_temp(rssItem.getStarred());
rssItem.setPubDate(pubDate);
rssItem.setRtl(rtl);
//Possible XSS fields
rssItem.setTitle(getStringOrDefault("title", "", e));
rssItem.setAuthor(getStringOrDefault("author", "", e));
rssItem.setLink(url);
rssItem.setEnclosureLink(enclosureLink);
rssItem.setEnclosureMime(enclosureMime);
rssItem.setMediaDescription(mediaDescription);
if(rssItem.getFingerprint() == null) {
rssItem.setFingerprint(UUID.randomUUID().toString());
}
// Calculate the size of the rss items - useful if users run into a SQLiteBlobTooBigException
// https://github.com/nextcloud/news-android/issues/887
int contentLength = content.length();
double sizeInMb = contentLength/1024d/1024d;
if(sizeInMb > 0.4) {
Log.w(TAG, "Massive rss item detected - " + content.length() + " chars / " + content.length() / 1024d / 1024d + "mb - url: " + rssItem.getLink());
// Trim string down to 500k characters
int maxLengthAllowed = 500000;
if(content.length() > maxLengthAllowed) {
Log.w(TAG, "Limiting rss item size to 500k characters - url:" + rssItem.getLink());
content = content.substring(0, maxLengthAllowed);
}
} else if(sizeInMb > 0.1) {
Log.w(TAG, "Large rss item detected - " + content.length() + " chars / " + content.length() / 1024d / 1024d + "mb - url: " + rssItem.getLink());
}
try {
// try fixing relative image links
content = ImageHandler.fixBrokenImageLinksInArticle(url, content);
} catch (Exception ex) {
ex.printStackTrace();
Log.e(TAG, "Error while fixing broken image links in article" + ex);
} catch (OutOfMemoryError error) {
error.printStackTrace();
Log.e(TAG, "OutOfMemoryError while fixing broken image links in article" + error);
Log.e(TAG, "OutOfMemoryError Article length:" + content.length());
}
rssItem.setBody(content);
String mediaThumbnail = getStringOrEmpty("mediaThumbnail", e); // Possible XSS Fields
if(mediaThumbnail.isEmpty()) {
List<String> images = ImageHandler.getImageLinksFromText(url, content);
if(images.size() > 0) {
Log.d(TAG, "extracted mediaThumbnail from body");
mediaThumbnail = images.get(0);
} else {
Log.d(TAG, "extracting mediaThumbnail from body failed - no images detected");
}
}
rssItem.setMediaThumbnail(mediaThumbnail);
return rssItem;
}
private static String getStringOrEmpty(String key, JsonObject jObj) {
return getStringOrDefault(key, "", jObj);
}
private static String getStringOrDefault(String key, String defaultValue, JsonObject jObj) {
if(jObj.has(key) && !jObj.get(key).isJsonNull()) {
return jObj.get(key).getAsString();
} else {
return defaultValue;
}
}
@SuppressWarnings("SameParameterValue")
private static Boolean getBooleanOrDefault(String key, Boolean defaultValue, JsonObject jObj) {
if(jObj.has(key) && !jObj.get(key).isJsonNull()) {
return jObj.get(key).getAsBoolean();
} else {
return defaultValue;
}
}
}
|
