patch [#25490] Fix for [#22096] Blender tries to open non-blend file

from Alexander Kuznetsov (alexk) with edits.

From the report:
Blender assumed that all files are .blend as retval = 0;

Now retval is initialized as file cannot be open (-1) for gzopen fail and directory case
retval = -2; is defined for not supported formats
This must be assigned before #ifdef WITH_PYTHON because this part can be missing
Finally retval = 0; if it is a .blend file

---
also made other edits.

- exotic.c's blend header checking was sloppy, didn't check data was actually read, only checked first 4 bytes and had a check for "blend.gz" extension which is unnecessary.
- use defines to help readability for BKE_read_exotic & BKE_read_file return values.
- no need to check for a NULL pointer before calling BKE_reportf(). (will just print to the console)
- print better reports when the file fails to load.

7 files changed, 53 insertions, 39 deletions

diff --git a/source/blender/blenkernel/BKE_blender.h b/source/blender/blenkernel/BKE_blender.h
index 886a11b126e..a5c5cadc4f6 100644
--- a/source/blender/blenkernel/BKE_blender.h
+++ b/source/blender/blenkernel/BKE_blender.h
@@ -51,6 +51,11 @@ struct Main;
 #define BLENDER_MINSUBVERSION	0
 
 int BKE_read_file(struct bContext *C, const char *filepath, struct ReportList *reports);
+
+#define BKE_READ_FILE_FAIL				0 /* no load */
+#define BKE_READ_FILE_OK				1 /* OK */
+#define BKE_READ_FILE_OK_USERPREFS		2 /* OK, and with new user settings */
+
 int BKE_read_file_from_memory(struct bContext *C, char* filebuf, int filelength, struct ReportList *reports);
 int BKE_read_file_from_memfile(struct bContext *C, struct MemFile *memfile, struct ReportList *reports);
 
diff --git a/source/blender/blenkernel/BKE_exotic.h b/source/blender/blenkernel/BKE_exotic.h
index 553e62ba578..ad59cd9c0a8 100644
--- a/source/blender/blenkernel/BKE_exotic.h
+++ b/source/blender/blenkernel/BKE_exotic.h
@@ -42,6 +42,13 @@ struct Scene;
  */
 int BKE_read_exotic(struct Scene *scene, const char *name);
 
+/* return codes */
+#define BKE_READ_EXOTIC_FAIL_PATH		-3 /* file format is not supported */
+#define BKE_READ_EXOTIC_FAIL_FORMAT		-2 /* file format is not supported */
+#define BKE_READ_EXOTIC_FAIL_OPEN		-1 /* Can't open the file */
+#define BKE_READ_EXOTIC_OK_BLEND		 0 /* .blend file */
+#define BKE_READ_EXOTIC_OK_OTHER		 1 /* other supported formats */
+
 void write_dxf(struct Scene *scene, char *str);
 void write_stl(struct Scene *scene, char *str);
 
diff --git a/source/blender/blenkernel/BKE_utildefines.h b/source/blender/blenkernel/BKE_utildefines.h
index 605a50e078b..f152ff16afb 100644
--- a/source/blender/blenkernel/BKE_utildefines.h
+++ b/source/blender/blenkernel/BKE_utildefines.h
@@ -203,8 +203,6 @@ do { \
 
 #define ID_NEW(a)		if( (a) && (a)->id.newid ) (a)= (void *)(a)->id.newid
 
-#define FORM MAKE_ID('F','O','R','M')
-
 #define BLEN MAKE_ID('B','L','E','N')
 #define DER_ MAKE_ID('D','E','R','_')
 #define V100 MAKE_ID('V','1','0','0')
diff --git a/source/blender/blenkernel/intern/blender.c b/source/blender/blenkernel/intern/blender.c
index b782d3a92a9..b218632e8e5 100644
--- a/source/blender/blenkernel/intern/blender.c
+++ b/source/blender/blenkernel/intern/blender.c
@@ -346,12 +346,6 @@ void BKE_userdef_free(void)
 	BLI_freelistN(&U.addons);
 }
 
-/* returns:
-   0: no load file
-   1: OK
-   2: OK, and with new user settings
-*/
-
 int BKE_read_file(bContext *C, const char *dir, ReportList *reports) 
 {
 	BlendFileData *bfd;
@@ -459,7 +453,7 @@ static int read_undosave(bContext *C, UndoElem *uel)
 	G.fileflags |= G_FILE_NO_UI;
 
 	if(UNDO_DISK) 
-		success= BKE_read_file(C, uel->str, NULL);
+		success= (BKE_read_file(C, uel->str, NULL) != BKE_READ_FILE_FAIL);
 	else
 		success= BKE_read_file_from_memfile(C, &uel->memfile, NULL);
 
diff --git a/source/blender/blenkernel/intern/exotic.c b/source/blender/blenkernel/intern/exotic.c
index e695c40401a..30cad2b6d2f 100644
--- a/source/blender/blenkernel/intern/exotic.c
+++ b/source/blender/blenkernel/intern/exotic.c
@@ -74,7 +74,7 @@
 #include "BKE_object.h"
 #include "BKE_material.h"
 #include "BKE_report.h"
-
+#include "BKE_exotic.h"
 #include "BKE_displist.h"
 #include "BKE_DerivedMesh.h"
 #include "BKE_curve.h"
@@ -458,49 +458,50 @@ int BKE_read_exotic(Scene *scene, const char *name)
 {
 	int len;
 	gzFile gzfile;
-	char str[32];
-	int *s0 = (int*) str;
-	int retval = 0;
+	int head[2];
+	int retval;
 
 	// make sure we're not trying to read a directory....
 
 	len= strlen(name);
-	if (name[len-1] !='/' && name[len-1] != '\\') {
+	if (ELEM(name[len-1], '/', '\\')) {
+		retval= BKE_READ_EXOTIC_FAIL_PATH;
+	}
+	else {
 		gzfile = gzopen(name,"rb");
 
-		if (NULL == gzfile ) {
-			//XXX error("Can't open file: %s", name);
-			retval= -1;
-		} else {
-			gzread(gzfile, str, 31);
+		if (gzfile == NULL) {
+			retval= BKE_READ_EXOTIC_FAIL_OPEN;
+		}
+		else {
+			len= gzread(gzfile, &head, sizeof(head));
 			gzclose(gzfile);
 
-			if ((*s0 != FORM) && (strncmp(str, "BLEN", 4) != 0) && !BLI_testextensie(name,".blend.gz")) {
-
+			if (len == sizeof(head) && (head[0] == BLEN && head[1] == DER_)) {
+				retval= BKE_READ_EXOTIC_OK_BLEND;
+			}
+			else {
 				//XXX waitcursor(1);
 				if(is_dxf(name)) {
 					dxf_read(scene, name);
-					retval = 1;
+					retval= BKE_READ_EXOTIC_OK_OTHER;
 				}
 				else if(is_stl(name)) {
 					if (is_stl_ascii(name))
 						read_stl_mesh_ascii(scene, name);
 					else
 						read_stl_mesh_binary(scene, name);
-					retval = 1;
+					retval= BKE_READ_EXOTIC_OK_OTHER;
 				}
-#ifdef WITH_PYTHON
-				// TODO: this should not be in the kernel...
-				else { // unknown format, call Python importloader 
-					/* pass */
+				else {
+					retval= BKE_READ_EXOTIC_FAIL_FORMAT;
 				}
-#endif /* WITH_PYTHON */
 				//XXX waitcursor(0);
 			}
 		}
 	}
 	
-	return (retval);
+	return retval;
 }
 
 
diff --git a/source/blender/windowmanager/intern/wm_files.c b/source/blender/windowmanager/intern/wm_files.c
index e7165598e38..22b10c8c618 100644
--- a/source/blender/windowmanager/intern/wm_files.c
+++ b/source/blender/windowmanager/intern/wm_files.c
@@ -276,7 +276,7 @@ void WM_read_file(bContext *C, const char *name, ReportList *reports)
 	retval= BKE_read_exotic(CTX_data_scene(C), name);
 	
 	/* we didn't succeed, now try to read Blender file */
-	if (retval== 0) {
+	if (retval == BKE_READ_EXOTIC_OK_BLEND) {
 		int G_f= G.f;
 		ListBase wmbase;
 
@@ -298,9 +298,9 @@ void WM_read_file(bContext *C, const char *name, ReportList *reports)
 		
 // XXX		mainwindow_set_filename_to_title(G.main->name);
 
-		if(retval==2) wm_init_userdef(C);	// in case a userdef is read from regular .blend
+		if(retval == BKE_READ_FILE_OK_USERPREFS) wm_init_userdef(C);	// in case a userdef is read from regular .blend
 		
-		if (retval!=0) {
+		if (retval != BKE_READ_FILE_FAIL) {
 			G.relbase_valid = 1;
 			if(!G.background) /* assume automated tasks with background, dont write recent file list */
 				write_history();
@@ -327,13 +327,22 @@ void WM_read_file(bContext *C, const char *name, ReportList *reports)
 		BKE_write_undo(C, "original");	/* save current state */
 		
 	}
-	else if(retval==1)
+	else if(retval == BKE_READ_EXOTIC_OK_OTHER)
 		BKE_write_undo(C, "Import file");
-	else if(retval == -1) {
-		if(reports)
-			BKE_reportf(reports, RPT_ERROR, "Can't read file: \"%s\", %s.", name, errno ? strerror(errno) : "Incompatible file format");
+	else if(retval == BKE_READ_EXOTIC_FAIL_OPEN) {
+		BKE_reportf(reports, RPT_ERROR, "Can't read file: \"%s\", %s.", name, errno ? strerror(errno) : "Unable to open the file");
 	}
-	
+	else if(retval == BKE_READ_EXOTIC_FAIL_FORMAT) {
+		BKE_reportf(reports, RPT_ERROR, "File format is not supported in file: \"%s\".", name);
+	}
+	else if(retval == BKE_READ_EXOTIC_FAIL_PATH) {
+		BKE_reportf(reports, RPT_ERROR, "File path invalid: \"%s\".", name);
+	}
+	else {
+		BKE_reportf(reports, RPT_ERROR, "Unknown error loading: \"%s\".", name);
+		BKE_assert(!"invalid 'retval'");
+	}
+
 	WM_cursor_wait(0);
 
 }
@@ -372,7 +381,7 @@ int WM_read_homefile(bContext *C, wmOperator *op)
 	wm_window_match_init(C, &wmbase); 
 	
 	if (!from_memory && BLI_exists(tstr)) {
-		success = BKE_read_file(C, tstr, NULL);
+		success = (BKE_read_file(C, tstr, NULL) != BKE_READ_FILE_FAIL);
 		
 		if(U.themes.first==NULL) {
 			printf("\nError: No valid startup.blend, fall back to built-in default.\n\n");
diff --git a/source/creator/creator.c b/source/creator/creator.c
index 92da0fe5beb..5b994be2c06 100644
--- a/source/creator/creator.c
+++ b/source/creator/creator.c
@@ -904,7 +904,7 @@ static int load_file(int UNUSED(argc), char **argv, void *data)
 
 		/*we successfully loaded a blend file, get sure that
 		pointcache works */
-		if (retval!=0) {
+		if (retval != BKE_READ_FILE_FAIL) {
 			wmWindowManager *wm= CTX_wm_manager(C);
 
 			/* special case, 2.4x files */