diff options
author | Douwe Maan <douwe@gitlab.com> | 2016-11-18 20:15:56 +0300 |
---|---|---|
committer | Douwe Maan <douwe@gitlab.com> | 2016-11-18 20:15:56 +0300 |
commit | 2717675fbc10c97904e6a3eebf498d2c53fe5ce5 (patch) | |
tree | 098245239581426ea1a1a917eed62bdb784905ff /spec | |
parent | 88479f7f3071c2d79447896deeea63fb78b175df (diff) | |
parent | e1b868307169b562a595b5cb41bda7e8b984600f (diff) |
Merge branch 'bugfix/html-only-mail' into 'master'
add parsing support for incoming html email
## What does this MR do?
Fixes #18388 by adding support for parsing HTML email
## Are there points in the code the reviewer needs to double check?
The new class, Gitlab::Email::HTMLParser, which needs to translate the HTML content to text and also delete replies, as they are not necessarily in the correct format to be caught by EmailReplyParser. The solution I found that should work for any HTML-formatted email is to remove all `<table>` and `<blockquote>` tags. Actual `<table>` elements (to be interpreted by markdown) should already be encoded with e.g. `<table>` - the only failure mode is if there is an *actual* HTML table in the content itself, which we wouldn't be able to support easily anyways.
The gem `html2text` traverses the HTML tree and outputs text - and markdown in the case of HTML links or images.
See merge request !7397
Diffstat (limited to 'spec')
-rw-r--r-- | spec/fixtures/emails/outlook_html.eml | 140 | ||||
-rw-r--r-- | spec/lib/gitlab/email/reply_parser_spec.rb | 4 |
2 files changed, 144 insertions, 0 deletions
diff --git a/spec/fixtures/emails/outlook_html.eml b/spec/fixtures/emails/outlook_html.eml new file mode 100644 index 00000000000..506d69efe83 --- /dev/null +++ b/spec/fixtures/emails/outlook_html.eml @@ -0,0 +1,140 @@ + +MIME-Version: 1.0 +Received: by 10.25.161.144 with HTTP; Tue, 7 Oct 2014 22:17:17 -0700 (PDT) +X-Originating-IP: [117.207.85.84] +In-Reply-To: <5434c8b52bb3a_623ff09fec70f049749@discourse-app.mail> +References: <topic/35@discourse.techapj.com> + <5434c8b52bb3a_623ff09fec70f049749@discourse-app.mail> +Date: Wed, 8 Oct 2014 10:47:17 +0530 +Delivered-To: arpit@techapj.com +Message-ID: <CAOJeqne=SJ_LwN4sb-0Y95ejc2OpreVhdmcPn0TnmwSvTCYzzQ@mail.gmail.com> +Subject: Re: [Discourse] [Meta] Welcome to techAPJ's Discourse! +From: Arpit Jalan <arpit@techapj.com> +To: Discourse <mail+e1c7f2a380e33840aeb654f075490bad@arpitjalan.com>Accept-Language: en-US +Content-Language: en-US +X-MS-Has-Attach: +X-MS-TNEF-Correlator: +x-originating-ip: [134.68.31.227] +Content-Type: multipart/alternative; + boundary="_000_B0DFE1BEB3739743BC9B639D0E6BC8FF217A6341IUMSSGMBX104ads_" +MIME-Version: 1.0 + +--_000_B0DFE1BEB3739743BC9B639D0E6BC8FF217A6341IUMSSGMBX104ads_ +Content-Type: text/html; charset="utf-8" +Content-Transfer-Encoding: base64 + +PGh0bWwgeG1sbnM6dj0idXJuOnNjaGVtYXMtbWljcm9zb2Z0LWNvbTp2bWwiIHhtbG5zOm89InVy +bjpzY2hlbWFzLW1pY3Jvc29mdC1jb206b2ZmaWNlOm9mZmljZSIgeG1sbnM6dz0idXJuOnNjaGVt +YXMtbWljcm9zb2Z0LWNvbTpvZmZpY2U6d29yZCIgeG1sbnM6bT0iaHR0cDovL3NjaGVtYXMubWlj +cm9zb2Z0LmNvbS9vZmZpY2UvMjAwNC8xMi9vbW1sIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcv +VFIvUkVDLWh0bWw0MCI+DQo8aGVhZD4NCjxtZXRhIGh0dHAtZXF1aXY9IkNvbnRlbnQtVHlwZSIg +Y29udGVudD0idGV4dC9odG1sOyBjaGFyc2V0PXV0Zi04Ij4NCjxtZXRhIG5hbWU9IkdlbmVyYXRv +ciIgY29udGVudD0iTWljcm9zb2Z0IFdvcmQgMTQgKGZpbHRlcmVkIG1lZGl1bSkiPg0KPCEtLVtp +ZiAhbXNvXT48c3R5bGU+dlw6KiB7YmVoYXZpb3I6dXJsKCNkZWZhdWx0I1ZNTCk7fQ0Kb1w6KiB7 +YmVoYXZpb3I6dXJsKCNkZWZhdWx0I1ZNTCk7fQ0Kd1w6KiB7YmVoYXZpb3I6dXJsKCNkZWZhdWx0 +I1ZNTCk7fQ0KLnNoYXBlIHtiZWhhdmlvcjp1cmwoI2RlZmF1bHQjVk1MKTt9DQo8L3N0eWxlPjwh +W2VuZGlmXS0tPjxzdHlsZT48IS0tDQovKiBGb250IERlZmluaXRpb25zICovDQpAZm9udC1mYWNl +DQoJe2ZvbnQtZmFtaWx5OkNhbGlicmk7DQoJcGFub3NlLTE6MiAxNSA1IDIgMiAyIDQgMyAyIDQ7 +fQ0KQGZvbnQtZmFjZQ0KCXtmb250LWZhbWlseTpUYWhvbWE7DQoJcGFub3NlLTE6MiAxMSA2IDQg +MyA1IDQgNCAyIDQ7fQ0KLyogU3R5bGUgRGVmaW5pdGlvbnMgKi8NCnAuTXNvTm9ybWFsLCBsaS5N +c29Ob3JtYWwsIGRpdi5Nc29Ob3JtYWwNCgl7bWFyZ2luOjBpbjsNCgltYXJnaW4tYm90dG9tOi4w +MDAxcHQ7DQoJZm9udC1zaXplOjEyLjBwdDsNCglmb250LWZhbWlseToiVGltZXMgTmV3IFJvbWFu +Iiwic2VyaWYiO30NCmE6bGluaywgc3Bhbi5Nc29IeXBlcmxpbmsNCgl7bXNvLXN0eWxlLXByaW9y +aXR5Ojk5Ow0KCWNvbG9yOmJsdWU7DQoJdGV4dC1kZWNvcmF0aW9uOnVuZGVybGluZTt9DQphOnZp +c2l0ZWQsIHNwYW4uTXNvSHlwZXJsaW5rRm9sbG93ZWQNCgl7bXNvLXN0eWxlLXByaW9yaXR5Ojk5 +Ow0KCWNvbG9yOnB1cnBsZTsNCgl0ZXh0LWRlY29yYXRpb246dW5kZXJsaW5lO30NCnANCgl7bXNv +LXN0eWxlLXByaW9yaXR5Ojk5Ow0KCW1zby1tYXJnaW4tdG9wLWFsdDphdXRvOw0KCW1hcmdpbi1y +aWdodDowaW47DQoJbXNvLW1hcmdpbi1ib3R0b20tYWx0OmF1dG87DQoJbWFyZ2luLWxlZnQ6MGlu +Ow0KCWZvbnQtc2l6ZToxMi4wcHQ7DQoJZm9udC1mYW1pbHk6IlRpbWVzIE5ldyBSb21hbiIsInNl +cmlmIjt9DQpzcGFuLkVtYWlsU3R5bGUxOA0KCXttc28tc3R5bGUtdHlwZTpwZXJzb25hbC1yZXBs +eTsNCglmb250LWZhbWlseToiQ2FsaWJyaSIsInNhbnMtc2VyaWYiOw0KCWNvbG9yOiMxRjQ5N0Q7 +fQ0KLk1zb0NocERlZmF1bHQNCgl7bXNvLXN0eWxlLXR5cGU6ZXhwb3J0LW9ubHk7DQoJZm9udC1m +YW1pbHk6IkNhbGlicmkiLCJzYW5zLXNlcmlmIjt9DQpAcGFnZSBXb3JkU2VjdGlvbjENCgl7c2l6 +ZTo4LjVpbiAxMS4waW47DQoJbWFyZ2luOjEuMGluIDEuMGluIDEuMGluIDEuMGluO30NCmRpdi5X +b3JkU2VjdGlvbjENCgl7cGFnZTpXb3JkU2VjdGlvbjE7fQ0KLS0+PC9zdHlsZT48IS0tW2lmIGd0 +ZSBtc28gOV0+PHhtbD4NCjxvOnNoYXBlZGVmYXVsdHMgdjpleHQ9ImVkaXQiIHNwaWRtYXg9IjEw +MjYiIC8+DQo8L3htbD48IVtlbmRpZl0tLT48IS0tW2lmIGd0ZSBtc28gOV0+PHhtbD4NCjxvOnNo +YXBlbGF5b3V0IHY6ZXh0PSJlZGl0Ij4NCjxvOmlkbWFwIHY6ZXh0PSJlZGl0IiBkYXRhPSIxIiAv +Pg0KPC9vOnNoYXBlbGF5b3V0PjwveG1sPjwhW2VuZGlmXS0tPg0KPC9oZWFkPg0KPGJvZHkgbGFu +Zz0iRU4tVVMiIGxpbms9ImJsdWUiIHZsaW5rPSJwdXJwbGUiPg0KPGRpdiBjbGFzcz0iV29yZFNl +Y3Rpb24xIj4NCjxwIGNsYXNzPSJNc29Ob3JtYWwiPjxzcGFuIHN0eWxlPSJmb250LXNpemU6MTEu +MHB0O2ZvbnQtZmFtaWx5OiZxdW90O0NhbGlicmkmcXVvdDssJnF1b3Q7c2Fucy1zZXJpZiZxdW90 +Oztjb2xvcjojMUY0OTdEIj5NaWNyb3NvZnQgT3V0bG9vayAyMDEwPG86cD48L286cD48L3NwYW4+ +PC9wPg0KPHAgY2xhc3M9Ik1zb05vcm1hbCI+PHNwYW4gc3R5bGU9ImZvbnQtc2l6ZToxMS4wcHQ7 +Zm9udC1mYW1pbHk6JnF1b3Q7Q2FsaWJyaSZxdW90OywmcXVvdDtzYW5zLXNlcmlmJnF1b3Q7O2Nv +bG9yOiMxRjQ5N0QiPjxvOnA+Jm5ic3A7PC9vOnA+PC9zcGFuPjwvcD4NCjxwIGNsYXNzPSJNc29O +b3JtYWwiPjxiPjxzcGFuIHN0eWxlPSJmb250LXNpemU6MTAuMHB0O2ZvbnQtZmFtaWx5OiZxdW90 +O1RhaG9tYSZxdW90OywmcXVvdDtzYW5zLXNlcmlmJnF1b3Q7Ij5Gcm9tOjwvc3Bhbj48L2I+PHNw +YW4gc3R5bGU9ImZvbnQtc2l6ZToxMC4wcHQ7Zm9udC1mYW1pbHk6JnF1b3Q7VGFob21hJnF1b3Q7 +LCZxdW90O3NhbnMtc2VyaWYmcXVvdDsiPiBtaWNoYWVsIFttYWlsdG86dGFsa0BvcGVubXJzLm9y +Z10NCjxicj4NCjxiPlNlbnQ6PC9iPiBNb25kYXksIE9jdG9iZXIgMTMsIDIwMTQgOTozOCBBTTxi +cj4NCjxiPlRvOjwvYj4gUG93ZXIsIENocmlzPGJyPg0KPGI+U3ViamVjdDo8L2I+IFtQTV0gWW91 +ciBwb3N0IGluICZxdW90O0J1cmdlcmhhdXM6IE5ldyByZXN0YXVyYW50IC8gbHVuY2ggdmVudWUm +cXVvdDs8bzpwPjwvbzpwPjwvc3Bhbj48L3A+DQo8cCBjbGFzcz0iTXNvTm9ybWFsIj48bzpwPiZu +YnNwOzwvbzpwPjwvcD4NCjxkaXY+DQo8dGFibGUgY2xhc3M9Ik1zb05vcm1hbFRhYmxlIiBib3Jk +ZXI9IjAiIGNlbGxzcGFjaW5nPSIwIiBjZWxscGFkZGluZz0iMCI+DQo8dGJvZHk+DQo8dHI+DQo8 +dGQgdmFsaWduPSJ0b3AiIHN0eWxlPSJwYWRkaW5nOjBpbiAwaW4gMGluIDBpbiI+PC90ZD4NCjx0 +ZCBzdHlsZT0icGFkZGluZzowaW4gMGluIDBpbiAwaW4iPg0KPHAgY2xhc3M9Ik1zb05vcm1hbCIg +c3R5bGU9Im1hcmdpbi1ib3R0b206MTguNzVwdCI+PGEgaHJlZj0iaHR0cDovL2NsLm9wZW5tcnMu +b3JnL3RyYWNrL2NsaWNrLzMwMDM5OTA1L3RhbGsub3Blbm1ycy5vcmc/cD1leUp6SWpvaWJHbFph +MVYwZVhoQ1kwMU1SVEZzVURKbVl6VlFNMFpsZWpFNElpd2lkaUk2TVN3aWNDSTZJbnRjSW5WY0lq +b3pNREF6T1Rrd05TeGNJblpjSWpveExGd2lkWEpzWENJNlhDSm9kSFJ3Y3pwY1hGd3ZYRnhjTDNS +aGJHc3ViM0JsYm0xeWN5NXZjbWRjWEZ3dmRYTmxjbk5jWEZ3dmJXbGphR0ZsYkZ3aUxGd2lhV1Jj +SWpwY0ltUTFZbU13TjJOa05EUmpaRFE0TUdNNFlUZzJNemxqWldJMU56Z3pZbVkyWENJc1hDSjFj +bXhmYVdSelhDSTZXMXdpWWpoa09EZzFNams1TnpkbVpqWTFaV1l5TlRFM09XUmlOR1l5TVdJM056 +RmpOemhqWmpoa09Gd2lYWDBpZlEiIHRhcmdldD0iX2JsYW5rIj48Yj48c3BhbiBzdHlsZT0iZm9u +dC1zaXplOjEwLjBwdDtmb250LWZhbWlseTomcXVvdDtUYWhvbWEmcXVvdDssJnF1b3Q7c2Fucy1z +ZXJpZiZxdW90Oztjb2xvcjojMDA2Njk5O3RleHQtZGVjb3JhdGlvbjpub25lIj5taWNoYWVsPC9z +cGFuPjwvYj48L2E+PGJyPg0KPHNwYW4gc3R5bGU9ImZvbnQtc2l6ZTo4LjVwdDtmb250LWZhbWls +eTomcXVvdDtUYWhvbWEmcXVvdDssJnF1b3Q7c2Fucy1zZXJpZiZxdW90Oztjb2xvcjojOTk5OTk5 +Ij5PY3RvYmVyIDEzPC9zcGFuPg0KPG86cD48L286cD48L3A+DQo8L3RkPg0KPC90cj4NCjx0cj4N +Cjx0ZCBjb2xzcGFuPSIyIiBzdHlsZT0icGFkZGluZzozLjc1cHQgMGluIDBpbiAwaW4iPg0KPHAg +Y2xhc3M9Ik1zb05vcm1hbCIgc3R5bGU9Im1hcmdpbi1ib3R0b206MTguNzVwdCI+PGEgaHJlZj0i +aHR0cDovL2NsLm9wZW5tcnMub3JnL3RyYWNrL2NsaWNrLzMwMDM5OTA1L3RhbGsub3Blbm1ycy5v +cmc/cD1leUp6SWpvaVVFUklTVU55UjNsVk1EZEJWVmhwV25SM1dXeDRNV05zVFc1Wklpd2lkaUk2 +TVN3aWNDSTZJbnRjSW5WY0lqb3pNREF6T1Rrd05TeGNJblpjSWpveExGd2lkWEpzWENJNlhDSm9k +SFJ3Y3pwY1hGd3ZYRnhjTDNSaGJHc3ViM0JsYm0xeWN5NXZjbWRjWEZ3dmRGeGNYQzlpZFhKblpY +Sm9ZWFZ6TFc1bGR5MXlaWE4wWVhWeVlXNTBMV3gxYm1Ob0xYWmxiblZsWEZ4Y0x6WTNNbHhjWEM4 +elhDSXNYQ0pwWkZ3aU9sd2laRFZpWXpBM1kyUTBOR05rTkRnd1l6aGhPRFl6T1dObFlqVTNPRE5p +WmpaY0lpeGNJblZ5YkY5cFpITmNJanBiWENKaU56WmlZamswWlRGaU56STVaVGsyWlRSbFpXTTRO +R1JtTWpRNE1ETXdZall5WVdZeU1HTTBYQ0pkZlNKOSI+PGI+PHNwYW4gc3R5bGU9ImNvbG9yOiMw +MDY2OTk7dGV4dC1kZWNvcmF0aW9uOm5vbmUiPmh0dHBzOi8vdGFsay5vcGVubXJzLm9yZy90L2J1 +cmdlcmhhdXMtbmV3LXJlc3RhdXJhbnQtbHVuY2gtdmVudWUvNjcyLzM8L3NwYW4+PC9iPjwvYT4N +CjxvOnA+PC9vOnA+PC9wPg0KPHAgc3R5bGU9Im1hcmdpbi10b3A6MGluIj5Mb29rcyBsaWtlIHlv +dXIgcmVwbHktYnktZW1haWwgd2Fzbid0IHByb2Nlc3NlZCBjb3JyZWN0bHkgYnkgb3VyIHNvZnR3 +YXJlLiBDYW4geW91IGxldCBtZSBrbm93IHdoYXQgdmVyc2lvbi9PUyBvZiB3aGF0IGVtYWlsIHBy +b2dyYW0geW91J3JlIHVzaW5nPyBXZSB3aWxsIHdhbnQgdG8gdHJ5IHRvIGZpeCB0aGUgYnVnLiA6 +c21pbGU6PG86cD48L286cD48L3A+DQo8cCBzdHlsZT0ibWFyZ2luLXRvcDowaW4iPlRoYW5rcyE8 +bzpwPjwvbzpwPjwvcD4NCjwvdGQ+DQo8L3RyPg0KPC90Ym9keT4NCjwvdGFibGU+DQo8ZGl2IGNs +YXNzPSJNc29Ob3JtYWwiIGFsaWduPSJjZW50ZXIiIHN0eWxlPSJ0ZXh0LWFsaWduOmNlbnRlciI+ +DQo8aHIgc2l6ZT0iMSIgd2lkdGg9IjEwMCUiIGFsaWduPSJjZW50ZXIiPg0KPC9kaXY+DQo8ZGl2 +Pg0KPHA+PHNwYW4gc3R5bGU9ImNvbG9yOiM2NjY2NjYiPlRvIHJlc3BvbmQsIHJlcGx5IHRvIHRo +aXMgZW1haWwgb3IgdmlzaXQgPGEgaHJlZj0iaHR0cDovL2NsLm9wZW5tcnMub3JnL3RyYWNrL2Ns +aWNrLzMwMDM5OTA1L3RhbGsub3Blbm1ycy5vcmc/cD1leUp6SWpvaWVYaDJWbnBGTUhSMU1uRm5a +RWR1TlhFd01GcFFPVlp0VFZvNElpd2lkaUk2TVN3aWNDSTZJbnRjSW5WY0lqb3pNREF6T1Rrd05T +eGNJblpjSWpveExGd2lkWEpzWENJNlhDSm9kSFJ3Y3pwY1hGd3ZYRnhjTDNSaGJHc3ViM0JsYm0x +eWN5NXZjbWRjWEZ3dmRGeGNYQzk1YjNWeUxYQnZjM1F0YVc0dFluVnlaMlZ5YUdGMWN5MXVaWGN0 +Y21WemRHRjFjbUZ1ZEMxc2RXNWphQzEyWlc1MVpWeGNYQzgyTnpSY1hGd3ZNVndpTEZ3aWFXUmNJ +anBjSW1RMVltTXdOMk5rTkRSalpEUTRNR000WVRnMk16bGpaV0kxTnpnelltWTJYQ0lzWENKMWNt +eGZhV1J6WENJNlcxd2lZamMyWW1JNU5HVXhZamN5T1dVNU5tVTBaV1ZqT0RSa1pqSTBPREF6TUdJ +Mk1tRm1NakJqTkZ3aVhYMGlmUSI+DQo8Yj48c3BhbiBzdHlsZT0iY29sb3I6IzAwNjY5OTt0ZXh0 +LWRlY29yYXRpb246bm9uZSI+aHR0cHM6Ly90YWxrLm9wZW5tcnMub3JnL3QveW91ci1wb3N0LWlu +LWJ1cmdlcmhhdXMtbmV3LXJlc3RhdXJhbnQtbHVuY2gtdmVudWUvNjc0LzE8L3NwYW4+PC9iPjwv +YT4gaW4geW91ciBicm93c2VyLjxvOnA+PC9vOnA+PC9zcGFuPjwvcD4NCjwvZGl2Pg0KPGRpdj4N +CjxwPjxzcGFuIHN0eWxlPSJjb2xvcjojNjY2NjY2Ij5UbyB1bnN1YnNjcmliZSBmcm9tIHRoZXNl +IGVtYWlscywgdmlzaXQgeW91ciA8YSBocmVmPSJodHRwOi8vY2wub3Blbm1ycy5vcmcvdHJhY2sv +Y2xpY2svMzAwMzk5MDUvdGFsay5vcGVubXJzLm9yZz9wPWV5SnpJam9pZFV4dVdsZzVWRmMwT1da +V1MwWTRiRmRMZG1seVdHc3hUVjl6SWl3aWRpSTZNU3dpY0NJNkludGNJblZjSWpvek1EQXpPVGt3 +TlN4Y0luWmNJam94TEZ3aWRYSnNYQ0k2WENKb2RIUndjenBjWEZ3dlhGeGNMM1JoYkdzdWIzQmxi +bTF5Y3k1dmNtZGNYRnd2YlhsY1hGd3ZjSEpsWm1WeVpXNWpaWE5jSWl4Y0ltbGtYQ0k2WENKa05X +SmpNRGRqWkRRMFkyUTBPREJqT0dFNE5qTTVZMlZpTlRjNE0ySm1ObHdpTEZ3aWRYSnNYMmxrYzF3 +aU9sdGNJbUk0TVdVd1pqQTFORFk1TkRNME56Z3lNMkZtTWpBMk5qRmpaamMzWkdOaU4yTmhZemRt +TWpKY0lsMTlJbjAiPg0KPGI+PHNwYW4gc3R5bGU9ImNvbG9yOiMwMDY2OTk7dGV4dC1kZWNvcmF0 +aW9uOm5vbmUiPnVzZXIgcHJlZmVyZW5jZXM8L3NwYW4+PC9iPjwvYT4uPG86cD48L286cD48L3Nw +YW4+PC9wPg0KPC9kaXY+DQo8L2Rpdj4NCjxwIGNsYXNzPSJNc29Ob3JtYWwiPjxpbWcgYm9yZGVy +PSIwIiB3aWR0aD0iMSIgaGVpZ2h0PSIxIiBpZD0iX3gwMDAwX2kxMDI2IiBzcmM9Imh0dHA6Ly9j +bC5vcGVubXJzLm9yZy90cmFjay9vcGVuLnBocD91PTMwMDM5OTA1JmFtcDtpZD1kNWJjMDdjZDQ0 +Y2Q0ODBjOGE4NjM5Y2ViNTc4M2JmNiI+PG86cD48L286cD48L3A+DQo8L2Rpdj4NCjwvYm9keT4N +CjwvaHRtbD4NCg== + +--_000_B0DFE1BEB3739743BC9B639D0E6BC8FF217A6341IUMSSGMBX104ads_-- diff --git a/spec/lib/gitlab/email/reply_parser_spec.rb b/spec/lib/gitlab/email/reply_parser_spec.rb index 6f8e9a4be64..c7a0139d32a 100644 --- a/spec/lib/gitlab/email/reply_parser_spec.rb +++ b/spec/lib/gitlab/email/reply_parser_spec.rb @@ -206,5 +206,9 @@ describe Gitlab::Email::ReplyParser, lib: true do it "properly renders email reply from MS Outlook client" do expect(test_parse_body(fixture_file("emails/outlook.eml"))).to eq("Microsoft Outlook 2010") end + + it "properly renders html-only email from MS Outlook" do + expect(test_parse_body(fixture_file("emails/outlook_html.eml"))).to eq("Microsoft Outlook 2010") + end end end |