Text plugin script updates: Better error handling, variable parsing, token caching for repeat parsing of the same document. Fixed joining of multiline statements and context detection.

4 files changed, 157 insertions, 66 deletions

diff --git a/release/scripts/bpymodules/BPyTextPlugin.py b/release/scripts/bpymodules/BPyTextPlugin.py
index 38bdab82a2d..2489c22f600 100644
--- a/release/scripts/bpymodules/BPyTextPlugin.py
+++ b/release/scripts/bpymodules/BPyTextPlugin.py
@@ -1,6 +1,7 @@
-import bpy, sys
+import bpy
 import __builtin__, tokenize
-from tokenize import generate_tokens
+from Blender.sys import time
+from tokenize import generate_tokens, TokenError
 # TODO: Remove the dependency for a full Python installation. Currently only the
 # tokenize module is required 
 
@@ -17,15 +18,33 @@ KEYWORDS = ['and', 'del', 'from', 'not', 'while', 'as', 'elif', 'global',
 			'raise', 'continue', 'finally', 'is', 'return', 'def', 'for',
 			'lambda', 'try' ]
 
+# Used to cache the return value of generate_tokens
+_token_cache = None
+_cache_update = 0
 
 def suggest_cmp(x, y):
-	"""Use this method when sorting a list for suggestions"""
+	"""Use this method when sorting a list of suggestions.
+	"""
 	
 	return cmp(x[0], y[0])
 
+def cached_generate_tokens(txt, since=1):
+	"""A caching version of generate tokens for multiple parsing of the same
+	document within a given timescale.
+	"""
+	
+	global _token_cache, _cache_update
+	
+	if _cache_update < time() - since:
+		txt.reset()
+		_token_cache = [g for g in generate_tokens(txt.readline)]
+		_cache_update = time()
+	return _token_cache
+
 def get_module(name):
-	"""Returns the module specified by its name. This module is imported and as
-	such will run any initialization code specified within the module."""
+	"""Returns the module specified by its name. The module itself is imported
+	by this method and, as such, any initialization code will be executed.
+	"""
 	
 	mod = __import__(name)
 	components = name.split('.')
@@ -34,11 +53,21 @@ def get_module(name):
 	return mod
 
 def is_module(m):
-	"""Taken from the inspect module of the standard Python installation"""
+	"""Taken from the inspect module of the standard Python installation.
+	"""
 	
 	return isinstance(m, type(bpy))
 
 def type_char(v):
+	"""Returns the character used to signify the type of a variable. Use this
+	method to identify the type character for an item in a suggestion list.
+	
+	The following values are returned:
+	  'm' if the parameter is a module
+	  'f' if the parameter is callable
+	  'v' if the parameter is variable or otherwise indeterminable
+	"""
+	
 	if is_module(v):
 		return 'm'
 	elif callable(v):
@@ -46,8 +75,8 @@ def type_char(v):
 	else: 
 		return 'v'
 
-def get_context(line, cursor):
-	"""Establishes the context of the cursor in the given line
+def get_context(txt):
+	"""Establishes the context of the cursor in the given Blender Text object
 	
 	Returns one of:
 	  NORMAL - Cursor is in a normal context
@@ -57,28 +86,43 @@ def get_context(line, cursor):
 	
 	"""
 	
+	l, cursor = txt.getCursorPos()
+	lines = txt.asLines()[:l+1]
+	
 	# Detect context (in string or comment)
 	in_str = 0			# 1-single quotes, 2-double quotes
-	for i in range(cursor):
-		if not in_str:
-			if line[i] == "'": in_str = 1
-			elif line[i] == '"': in_str = 2
-			elif line[i] == '#': return 3 # In a comment so quit
+	for line in lines:
+		if l == 0:
+			end = cursor
 		else:
-			if in_str == 1:
-				if line[i] == "'":
-					in_str = 0
-					# In again if ' escaped, out again if \ escaped, and so on
-					for a in range(1, i+1):
-						if line[i-a] == '\\': in_str = 1-in_str
-						else: break
-			elif in_str == 2:
-				if line[i] == '"':
-					in_str = 0
-					# In again if " escaped, out again if \ escaped, and so on
-					for a in range(1, i+1):
-						if line[i-a] == '\\': in_str = 2-in_str
-						else: break
+			end = len(line)
+			l -= 1
+		
+		# Comments end at new lines
+		if in_str == 3:
+			in_str = 0
+		
+		for i in range(end):
+			if in_str == 0:
+				if line[i] == "'": in_str = 1
+				elif line[i] == '"': in_str = 2
+				elif line[i] == '#': in_str = 3
+			else:
+				if in_str == 1:
+					if line[i] == "'":
+						in_str = 0
+						# In again if ' escaped, out again if \ escaped, and so on
+						for a in range(i-1, -1, -1):
+							if line[a] == '\\': in_str = 1-in_str
+							else: break
+				elif in_str == 2:
+					if line[i] == '"':
+						in_str = 0
+						# In again if " escaped, out again if \ escaped, and so on
+						for a in range(i-1, -1, -1):
+							if line[i-a] == '\\': in_str = 2-in_str
+							else: break
+		
 	return in_str
 
 def current_line(txt):
@@ -101,9 +145,10 @@ def current_line(txt):
 	
 	# Join later lines while there is an explicit joining character
 	i = lineindex
-	while i < len(lines)-1 and line[i].rstrip().endswith('\\'):
+	while i < len(lines)-1 and lines[i].rstrip().endswith('\\'):
 		later = lines[i+1].strip()
 		line = line + ' ' + later[:-1]
+		i += 1
 	
 	return line, cursor
 
@@ -134,9 +179,8 @@ def get_imports(txt):
 	# strings open or there are other syntax errors. For now we return an empty
 	# dictionary until an alternative parse method is implemented.
 	try:
-		txt.reset()
-		tokens = generate_tokens(txt.readline)
-	except:
+		tokens = cached_generate_tokens(txt)
+	except TokenError:
 		return dict()
 	
 	imports = dict()
@@ -191,8 +235,7 @@ def get_imports(txt):
 			# Handle special case of 'import *'
 			if impname == '*':
 				parent = get_module(fromname)
-				for symbol, attr in parent.__dict__.items():
-					imports[symbol] = attr
+				imports.update(parent.__dict__)
 				
 			else:
 				# Try importing the name as a module
@@ -202,12 +245,12 @@ def get_imports(txt):
 					else:
 						module = get_module(impname)
 					imports[symbol] = module
-				except:
+				except (ImportError, ValueError, AttributeError, TypeError):
 					# Try importing name as an attribute of the parent
 					try:
 						module = __import__(fromname, globals(), locals(), [impname])
 						imports[symbol] = getattr(module, impname)
-					except:
+					except (ImportError, ValueError, AttributeError, TypeError):
 						pass
 			
 			# More to import from the same module?
@@ -219,7 +262,6 @@ def get_imports(txt):
 	
 	return imports
 
-
 def get_builtins():
 	"""Returns a dictionary of built-in modules, functions and variables."""
 	
@@ -235,9 +277,8 @@ def get_defs(txt):
 	
 	# See above for problems with generate_tokens
 	try:
-		txt.reset()
-		tokens = generate_tokens(txt.readline)
-	except:
+		tokens = cached_generate_tokens(txt)
+	except TokenError:
 		return dict()
 	
 	defs = dict()
@@ -269,3 +310,37 @@ def get_defs(txt):
 				step = 0
 		
 	return defs
+
+def get_vars(txt):
+	"""Returns a dictionary of variable names found in the specified Text
+	object. This method locates all names followed directly by an equal sign:
+	'a = ???' or indirectly as part of a tuple/list assignment or inside a
+	'for ??? in ???:' block.
+	"""
+	
+	# See above for problems with generate_tokens
+	try:
+		tokens = cached_generate_tokens(txt)
+	except TokenError:
+		return []
+	
+	vars = []
+	accum = [] # Used for tuple/list assignment
+	foring = False
+	
+	for type, string, start, end, line in tokens:
+		
+		# Look for names
+		if string == 'for':
+			foring = True
+		if string == '=' or (foring and string == 'in'):
+			vars.extend(accum)
+			accum = []
+			foring = False
+		elif type == tokenize.NAME:
+			accum.append(string)
+		elif not string in [',', '(', ')', '[', ']']:
+			accum = []
+			foring = False
+		
+	return vars
diff --git a/release/scripts/textplugin_imports.py b/release/scripts/textplugin_imports.py
index af335eb5418..1773427bb01 100644
--- a/release/scripts/textplugin_imports.py
+++ b/release/scripts/textplugin_imports.py
@@ -13,7 +13,7 @@ try:
 	import bpy, sys
 	from BPyTextPlugin import *
 	OK = True
-except:
+except ImportError:
 	pass
 
 def main():
@@ -21,7 +21,7 @@ def main():
 	line, c = current_line(txt)
 	
 	# Check we are in a normal context
-	if get_context(line, c) != 0:
+	if get_context(txt) != 0:
 		return
 	
 	pos = line.rfind('from ', 0, c)
@@ -30,7 +30,7 @@ def main():
 	if pos == -1:
 		# Check instead for straight 'import'
 		pos2 = line.rfind('import ', 0, c)
-		if pos2 != -1 and pos2 == c-7:
+		if pos2 != -1 and (pos2 == c-7 or (pos2 < c-7 and line[c-2]==',')):
 			items = [(m, 'm') for m in sys.builtin_module_names]
 			items.sort(cmp = suggest_cmp)
 			txt.suggest(items, '')
@@ -54,7 +54,7 @@ def main():
 			between = line[pos+5:pos2-1].strip()
 			try:
 				mod = get_module(between)
-			except:
+			except ImportError:
 				print 'Module not found:', between
 				return
 			
diff --git a/release/scripts/textplugin_membersuggest.py b/release/scripts/textplugin_membersuggest.py
index d1ab588ba86..57c920c2bf9 100644
--- a/release/scripts/textplugin_membersuggest.py
+++ b/release/scripts/textplugin_membersuggest.py
@@ -13,7 +13,7 @@ try:
 	import bpy
 	from BPyTextPlugin import *
 	OK = True
-except:
+except ImportError:
 	OK = False
 
 def main():
@@ -21,7 +21,7 @@ def main():
 	(line, c) = current_line(txt)
 	
 	# Check we are in a normal context
-	if get_context(line, c) != NORMAL:
+	if get_context(txt) != NORMAL:
 		return
 	
 	pre = get_targets(line, c)
@@ -43,21 +43,24 @@ def main():
 	try:
 		for name in pre[1:-1]:
 			obj = getattr(obj, name)
-	except:
+	except AttributeError:
 		print "Attribute not found '%s' in '%s'" % (name, '.'.join(pre))
 		return
 	
 	try:
 		attr = obj.__dict__.keys()
-	except:
+	except AttributeError:
 		attr = dir(obj)
 	
 	for k in attr:
-		v = getattr(obj, k)
-		if is_module(v): t = 'm'
-		elif callable(v): t = 'f'
-		else: t = 'v'
-		list.append((k, t))
+		try:
+			v = getattr(obj, k)
+			if is_module(v): t = 'm'
+			elif callable(v): t = 'f'
+			else: t = 'v'
+			list.append((k, t))
+		except (AttributeError, TypeError): # Some attributes are not readable
+			pass
 	
 	if list != []:
 		list.sort(cmp = suggest_cmp)
diff --git a/release/scripts/textplugin_suggest.py b/release/scripts/textplugin_suggest.py
index 8e14dffca9c..770d2759bcc 100644
--- a/release/scripts/textplugin_suggest.py
+++ b/release/scripts/textplugin_suggest.py
@@ -12,36 +12,46 @@ try:
 	import bpy
 	from BPyTextPlugin import *
 	OK = True
-except:
+except ImportError:
 	OK = False
 
+def check_membersuggest(line, c):
+	pos = line.rfind('.', 0, c)
+	if pos == -1:
+		return False
+	for s in line[pos+1:c]:
+		if not s.isalnum() and not s == '_':
+			return False
+	return True
+
+def check_imports(line, c):
+	if line.rfind('import ', 0, c) == c-7:
+		return True
+	if line.rfind('from ', 0, c) == c-5:
+		return True
+	return False
+
 def main():
 	txt = bpy.data.texts.active
 	(line, c) = current_line(txt)
 	
 	# Check we are in a normal context
-	if get_context(line, c) != NORMAL:
+	if get_context(txt) != NORMAL:
 		return
 	
-	# Check that which precedes the cursor and perform the following:
-	# Period(.)				- Run textplugin_membersuggest.py
-	# 'import' or 'from'	- Run textplugin_imports.py
-	# Other                 - Continue this script (global suggest)
-	pre = get_targets(line, c)
-	
-	count = len(pre)
+	# Check the character preceding the cursor and execute the corresponding script
 	
-	if count > 1: # Period found
+	if check_membersuggest(line, c):
 		import textplugin_membersuggest
-		textplugin_membersuggest.main()
 		return
-	# Look for 'import' or 'from'
-	elif line.rfind('import ', 0, c) == c-7 or line.rfind('from ', 0, c) == c-5:
+	
+	elif check_imports(line, c):
 		import textplugin_imports
-		textplugin_imports.main()
 		return
 	
+	# Otherwise we suggest globals, keywords, etc.
 	list = []
+	pre = get_targets(line, c)
 	
 	for k in KEYWORDS:
 		list.append((k, 'k'))
@@ -55,6 +65,9 @@ def main():
 	for k, v in get_defs(txt).items():
 		list.append((k, 'f'))
 	
+	for k in get_vars(txt):
+		list.append((k, 'v'))
+	
 	list.sort(cmp = suggest_cmp)
 	txt.suggest(list, pre[-1])