Add partial RosterX support (add only, for now)

Add groups support for roster contacts (via RosterX feature)
Code cleanup & fixes in groupchats extension
Custom registration forms (not done yet, check out library/forms.py)
Little refactoring in modulemanager library
WARNING: This commit may contain breaking changes. It may break
especially groupchats, module management, settings management,
registration process. DO NOT UPDATE to this release unless you're a pro.

8 files changed, 118 insertions, 125 deletions

diff --git a/extensions/groupchats.py b/extensions/groupchats.py
index cf28538..f1b736a 100644
--- a/extensions/groupchats.py
+++ b/extensions/groupchats.py
@@ -29,22 +29,14 @@ except ImportError:
 	mod_xhtml = None
 
 
-def sendIQ(chat, attr, data, afrls, role, jidFrom, reason=None, cb=None, args={}):
+def setAffiliation(chat, role, jid, jidFrom=TransportID, reason=None):
 	stanza = xmpp.Iq("set", to=chat, frm=jidFrom)
 	query = xmpp.Node("query", {"xmlns": xmpp.NS_MUC_ADMIN})
-	arole = query.addChild("item", {attr: data, afrls: role})
+	arole = query.addChild("item", {"jid": jid, "affiliation": role})
 	if reason:
 		arole.setTagData("reason", reason)
-	stanza.addChild(node = query)
-	sender(Component, stanza, cb, args)
-
-
-def makeMember(chat, jid, jidFrom, cb=None, args={}):
-	sendIQ(chat, "jid", jid, "affiliation", "member", jidFrom, cb=cb, args=args)
-
-
-def makeOutcast(chat, jid, jidFrom, reason=None, cb=None, args={}):
-	sendIQ(chat, "jid", jid, "affiliation", "outcast", jidFrom, reason=reason, cb=cb, args=args)
+	stanza.addChild(node=query)
+	sender(Component, stanza)
 
 
 def inviteUser(chat, jidTo, jidFrom, name):
@@ -192,13 +184,13 @@ class Chat(object):
 			self.raw_users = vkChat.get("users")
 
 		name = "@%s" % TransportID
-		makeMember(chat, user.source, TransportID)
+		setAffiliation(chat, "member", user.source)
 		if not self.invited:
 			inviteUser(chat, user.source, TransportID, user.vk.getUserData(self.owner)["name"])
 			self.invited = True
 		logger.debug("groupchats: user has been invited to chat %s (jid: %s)" % (chat, user.source))
 		chatMessage(chat, self.topic, TransportID, True, self.creation_date)
-		joinChat(chat, name, TransportID, "Lost in time.") ## let's rename ourself
+		joinChat(chat, name, TransportID, "Lost in time.") ## let's rename ourselves
 		self.users[TransportID] = {"name": name, "jid": TransportID}
 
 	def update(self, userObject, vkChat):
@@ -213,28 +205,27 @@ class Chat(object):
 			if users:
 				all_users = users.get("users", [])
 
-		for user in all_users:
-			## checking if there new users we didn't join yet
-			if not user in self.users.keys():
-				logger.debug("groupchats: user %s has joined the chat %s (jid: %s)" % (user, self.jid, userObject.source))
-				jid = vk2xmpp(user)
+		old_users = self.users.keys()
+		buddies = all_users + old_users
+		if TransportID in buddies:
+			buddies.remove(TransportID)
+
+		for user in buddies:
+			jid = vk2xmpp(user)
+			if user not in old_users:
+				logger.debug("groupchats: user %s has joined the chat %s (jid: %s)",
+					user, self.jid, userObject.source)
+
 				name = userObject.vk.getUserData(user)["name"]
 				self.users[int(user)] = {"name": name, "jid": jid}
-				makeMember(self.jid, jid, TransportID)
+				setAffiliation(self.jid, "member", jid)
 				joinChat(self.jid, name, jid) 
 
-		for user in self.users.keys():
-			## checking if there are old users we have to remove
-			if not user in all_users and user != TransportID:
-				logger.debug("groupchats: user %s has left the chat %s (jid: %s)" %\
-							(user, self.jid, userObject.source))
-				leaveChat(self.jid, vk2xmpp(user))
+			elif user not in all_users:
+				logger.debug("groupchats: user %s has left the chat %s (jid: %s)",
+					user, self.jid, userObject.source)
+				leaveChat(self.jid, jid)
 				del self.users[user]
-				if user == userObject.vk.userID:
-					logger.warning("groupchats: user %s has left the chat %s, so the chat would be EXTERMINATED (jid: %s)" %\
-								 	(user, self.jid, userObject.source))
-					self.setConfig(self.jid, TransportID, exterminate=True) ## exterminate the chats when user leave conference or just go offline?
-					del userObject.chats[self.jid]
 
 		topic = vkChat["title"]
 		if topic != self.topic:
@@ -252,10 +243,11 @@ class Chat(object):
 		if exterminate:
 			query.addChild("destroy")
 		else:
-			form = utils.buildDataForm(fields = [{"var": "FORM_TYPE", "type": "hidden", "value": xmpp.NS_MUC_ROOMCONFIG},
-											 {"var": "muc#roomconfig_membersonly", "type": "boolean", "value": "1"},
-											 {"var": "muc#roomconfig_publicroom", "type": "boolean", "value": "0"},
-											 {"var": "muc#roomconfig_whois", "value": "anyone"}], type="submit")
+			form = utils.buildDataForm(fields=[{"var": "FORM_TYPE", "type": "hidden", "value": xmpp.NS_MUC_ROOMCONFIG},
+				{"var": "muc#roomconfig_membersonly", "type": "boolean", "value": "1"},
+				{"var": "muc#roomconfig_publicroom", "type": "boolean", "value": "0"},
+				{"var": "muc#roomconfig_whois", "value": "anyone"}], 
+				type="submit")
 			query.addChild(node=form)
 		sender(Component, iq, cb, args)
 
@@ -267,10 +259,11 @@ class Chat(object):
 		if xmpp.isResultNode(stanza):
 			self.created = True
 			logger.debug("groupchats: stanza \"result\" received from %s,"\
-			 			 "continuing initialization (jid: %s)" % (chat, user.source))
+			 			 "continuing initialization (jid: %s)", chat, user.source)
 			utils.execute(self.initialize, (user, chat))
 		else:
-			logger.error("groupchats: couldn't set room %s config, the answer is: %s (jid: %s)" % (chat, str(stanza), user.source))
+			logger.error("groupchats: couldn't set room %s config, the answer is: %s (jid: %s)",
+				chat, str(stanza), user.source)
 
 	# here is a possibility to get messed up if many messages were sent before we created the chat 
 	# we have to send the messages immendiately as soon as possible, so delay can mess the messages up
@@ -311,7 +304,7 @@ class Chat(object):
 		Split the source and returns required parts
 		"""
 		node, domain = source.split("@")
-		if "_chat#" in node: ## Custom chat name?
+		if "_chat#" in node:  # Custom chat name?
 			creator, id = node.split("_chat#")
 		else:
 			return (None, None, None)
@@ -325,18 +318,18 @@ class Chat(object):
 		user = None
 		jid = None
 		creator, id, domain = cls.getParts(source)
-		if domain == ConferenceServer and creator: ## we will ignore zero-id
-			jid = getJIDByID(id)
+		if domain == ConferenceServer and creator:
+			jid = cls.getJIDByID(id)
 		if not jid:
-			jid  = runDatabaseQuery("select user from groupchats where jid=?", (source,), many=False, semph=None)
+			jid = runDatabaseQuery("select user from groupchats where jid=?", (source,), many=False, semph=None)
 			if jid:
 				jid = jid[0]
 		if jid and jid in Transport:
 			user = Transport[jid]
 		return user
 
-	@classmethod
-	def getJIDByID(cls, id):
+	@staticmethod
+	def getJIDByID(id):
 		for key, value in Transport.iteritems():
 			if key == id:
 				return value
@@ -370,21 +363,21 @@ def incomingChatMessageHandler(msg):
 				if source in getattr(user, "chats", {}):
 					owner_nickname = user.chats[source].owner_nickname
 				if not owner_nickname:
-					owner_nickname = runDatabaseQuery("select nick from groupchats where jid=?",\
-													(source,), many=False)[0]
+					owner_nickname = runDatabaseQuery("select nick from groupchats where jid=?",
+						(source,), many=False)[0]
 				# None of “normal” clients will send messages with timestamp
 				# If we do (as we set in force_vk_date_group), then the message received from a user
 				# If we don't and nick (as in settings) is tied to the chat, then we can determine who sent the message
 				send = ((nick == owner_nickname and user.settings.tie_chat_to_nickname)
-						or user.settings.force_vk_date_group)
+					or user.settings.force_vk_date_group)
 
 				if html and html.getTag("body"): ## XHTML-IM!
 					logger.debug("groupchats: fetched xhtml image (jid: %s)" % source)
 					try:
-						xhtml = mod_xhtml.parseXHTML(user, html, source, source, "chat_id")
+						mod_xhtml.parseXHTML(user, html, source, source, "chat_id")
 					except Exception:
-						xhtml = False
-					if xhtml:
+						pass
+					else:
 						# Don't send a message if there's an image
 						raise xmpp.NodeProcessed()
 				if send:
@@ -446,7 +439,7 @@ def handleChatPresences(source, prs):
 				id = vk2xmpp(jid)
 				if id != TransportID and id not in chat.users.keys():
 					if (time.gmtime().tm_mon, time.gmtime().tm_mday) == (4, 1):
-						makeOutcast(source, jid, TransportID, _("Get the hell outta here!"))
+						setAffiliation(source, "outcast", jid, reason=_("Get the hell outta here!"))
 					else:
 						leaveChat(source, jid, _("I am not welcomed here"))
 
@@ -564,7 +557,7 @@ if isdef("ConferenceServer") and ConferenceServer:
 	logger.info("extension groupchats is loaded")
 
 else:
-	del sendIQ, makeMember, makeOutcast, inviteUser, joinChat, leaveChat, \
+	del setAffiliation, inviteUser, joinChat, leaveChat, \
 		outgoingChatMessageHandler, chatMessage, Chat, \
 		incomingChatMessageHandler, handleChatErrors, handleChatPresences, exterminateChats, \
 		cleanTheChatsUp, initChatExtension
diff --git a/gateway.py b/gateway.py
index 5045298..2b9531e 100644
--- a/gateway.py
+++ b/gateway.py
@@ -196,6 +196,7 @@ escape = re.compile("|".join(unichr(x) for x in badChars),
 sortMsg = lambda first, second: first.get("mid", 0) - second.get("mid", 0)
 require = lambda name: os.path.exists("extensions/%s.py" % name)
 isdef = lambda var: var in globals()
+findListByID = lambda id, list: [key for key in list if key["lid"] == id]
 
 
 class VK(object):
@@ -212,6 +213,7 @@ class VK(object):
 		self.pollInitialzed = False
 		self.online = False
 		self.userID = 0
+		self.lists = []
 		self.friends_fields = set(["screen_name"])
 		logger.debug("VK.__init__ with number:%s from jid:%s", number, source)
 
@@ -391,7 +393,7 @@ class VK(object):
 		logger.debug("VK: user %s has left", self.source)
 		self.online = False
 		utils.runThread(executeHandlers, ("evt06", (self,)))
-		utils.runThread(self.method, ("account.setOffline", None, True, True))
+		utils.runThread(self.method, ("account.setOffline", None, True))
 
 	def getFriends(self, fields=None):
 		"""
@@ -408,7 +410,7 @@ class VK(object):
 			online = friend["online"]
 			name = escape("", str.join(chr(32), (friend["first_name"],
 				friend["last_name"])))
-			friends[uid] = {"name": name, "online": online}
+			friends[uid] = {"name": name, "online": online, "lists": friend["lists"]}
 			for key in fields:
 				friends[uid][key] = friend.get(key)
 		return friends
@@ -427,9 +429,15 @@ class VK(object):
 		"""
 		Gets user id
 		"""
-		self.userID = self.method("execute.getUserID")
+		if not self.userID:
+			self.userID = self.method("execute.getUserID")
 		return self.userID
 
+	def getLists(self):
+		if not self.lists:
+			self.lists = self.method("friends.getLists")
+		return self.lists
+
 	def getUserData(self, uid, fields=None):
 		"""
 		Gets user data. Such as name, photo, etc
@@ -568,6 +576,12 @@ class User(object):
 			self.friends = self.vk.getFriends()
 		return self.vk.online
 
+	def markRosterSet(self, cl=None, stanza=None):
+		self.rosterSet = True
+		runDatabaseQuery("update users set rosterSet=? where jid=?",
+			(self.rosterSet, self.source), True)
+
+
 	def initialize(self, force=False, send=True, resource=None, raise_exc=False):
 		"""
 		Initializes user after self.connect() is done:
@@ -588,7 +602,8 @@ class User(object):
 		if self.friends and not self.rosterSet or force:
 			logger.debug("User: sending subscription presence with force:%s (jid: %s)",
 				force, self.source)
-			self.sendSubPresence(self.friends)
+			import rostermanager
+			rostermanager.Roster.checkRosterx(self, resource)
 		if send:
 			self.sendInitPresence()
 		if resource:
@@ -636,7 +651,7 @@ class User(object):
 
 	def sendSubPresence(self, dist=None):
 		"""
-		Sends subsribe presence to self.source
+		Sends subscribe presence to self.source
 		Parameteres:
 			dist: friends list
 		"""
@@ -645,9 +660,7 @@ class User(object):
 			sendPresence(self.source, vk2xmpp(uid), "subscribe", value["name"])
 		sendPresence(self.source, TransportID, "subscribe", IDENTIFIER["name"])
 		if dist:
-			self.rosterSet = True
-			runDatabaseQuery("update users set rosterSet=? where jid=?",
-				(self.rosterSet, self.source), True)
+			self.markRosterSet()
 
 	def sendMessages(self, init=False):
 		"""
diff --git a/library/modulemanager.py b/library/modulemanager.py
index bf6ad55..d590802 100644
--- a/library/modulemanager.py
+++ b/library/modulemanager.py
@@ -20,20 +20,20 @@ def proxy(func):
 			for (handler, typ, ns, makefirst) in args:
 				if isinstance(ns, list):
 					while ns:
-						func(type, handler, typ, ns.pop(), makefirst)
+						func(type, handler, typ, ns.pop(), makefirst=makefirst)
 				else:
-					func(type, handler, typ, ns, makefirst)
+					func(type, handler, typ, ns, makefirst=makefirst)
 	return wrapper
 
 
 @proxy
-def register(type, handler, typ, ns, makefirst=False):
-	Component.RegisterHandler(type, handler, typ, ns, makefirst=makefirst)
+def register(*args, **kwargs):
+	Component.RegisterHandler(*args, **kwargs)
 
 
 @proxy
-def unregister(type, handler, typ, ns, _=0):
-	Component.UnregisterHandler(type, handler, typ, ns)
+def unregister(*args, **kwargs):
+	Component.UnregisterHandler(*args)
 
 
 def addFeatures(features, list=TransportFeatures):
@@ -89,31 +89,29 @@ class ModuleManager:
 		return modules
 
 	@classmethod
-	def reload(cls, name):
-		if name in sys.modules:
-			module = sys.modules[name]
-			cls.__unregister(module)
-			return reload(module)
+	def __load(cls, name, reload=False):
+		try:
+			if reload:
+				module = sys.modules[name]
+				cls.__unregister(module)
+				module = reload(module)
+			else:
+				module = __import__(name, globals(), locals())
+		except Exception:
+			crashLog("modulemanager.load")
+			module = None
+		return module
 
 	@classmethod
 	def load(cls, list=[]):
 		result = []
 		errors = []
 		for name in list:
-			if name in cls.loaded:
-				try:
-					module = cls.reload(name)
-				except Exception:
-					crashLog("modulemanager.reload")
-					errors.append(name)
-					continue
-			else:
-				try:
-					module = __import__(name, globals(), locals())
-				except Exception:
-					crashLog("modulemanager.load")
-					errors.append(name)
-					continue
+			loaded = name in cls.loaded
+			module = cls.__load(name, loaded)
+			if not module:
+				errors.append(name)
+				continue
 
 			result.append(name)
 			cls.__register(module)
diff --git a/library/settings.py b/library/settings.py
index e8b4418..170ccb2 100644
--- a/library/settings.py
+++ b/library/settings.py
@@ -10,6 +10,7 @@ __author__ = "mrDoctorWho <mrdoctorwho@gmail.com>"
 
 from __main__ import settingsDir, rFile, wFile
 from copy import deepcopy
+import os
 
 GLOBAL_USER_SETTINGS = {"keep_online": {"label": "Keep my status online",
 										"value": 1},
diff --git a/library/utils.py b/library/utils.py
index 5cca721..ac4a439 100644
--- a/library/utils.py
+++ b/library/utils.py
@@ -3,7 +3,7 @@
 # © simpleApps, 2014.
 
 """
-Contain useful functions which used across the modules
+Contains useful functions which used across the modules
 """
 
 import threading
@@ -60,6 +60,7 @@ def runThread(func, args=(), name=None, att=3, delay=0):
 		if att:
 			return runThread(func, args, name, (att - 1), delay)
 		crashLog("runThread.%s" % name)
+	return thr
 
 
 def safe(func):
diff --git a/library/vkapi.py b/library/vkapi.py
index 4af0bb3..b1b45f8 100644
--- a/library/vkapi.py
+++ b/library/vkapi.py
@@ -300,7 +300,7 @@ class APIBinding(object):
 
 		self.RIP = RequestProcessor()
 
-	def method(self, method, values=None, nodecode=False):
+	def method(self, method, values=None):
 		"""
 		Issues the VK method
 		Parameters:
diff --git a/modules/mod_iq_register.py b/modules/mod_iq_register.py
index a74e717..4718e49 100644
--- a/modules/mod_iq_register.py
+++ b/modules/mod_iq_register.py
@@ -5,13 +5,12 @@
 from __main__ import *
 from __main__ import _
 
-URL_ACCEPT_APP = URL_ACCEPT_APP % VK_ACCESS
-
 
 def initializeUser(user, cl, iq):
 	source = user.source
 	result = iq.buildReply("result")
 	connect = False
+	resource = iq.getFrom().getResource()
 	try:
 		connect = user.connect(True)
 	except (api.TokenError, api.AuthError) as e:
@@ -19,10 +18,10 @@ def initializeUser(user, cl, iq):
 	else:
 		if connect:
 			try:
-				user.initialize()
+				user.initialize(resource=resource)
 			except api.CaptchaNeeded:
 				user.vk.captchaChallenge()
-			except Exception: ## is there could be any other exception?
+			except Exception:
 				crashLog("user.init")
 				result = utils.buildIQError(iq, xmpp.ERR_BAD_REQUEST, _("Initialization failed."))
 			else:
@@ -32,6 +31,7 @@ def initializeUser(user, cl, iq):
 			result = utils.buildIQError(iq, xmpp.ERR_BAD_REQUEST, _("Incorrect password or access token!"))
 	sender(cl, result)
 
+import forms
 
 def register_handler(cl, iq):
 	jidTo = iq.getTo()
@@ -49,21 +49,8 @@ def register_handler(cl, iq):
 
 	if destination == TransportID:
 		if iType == "get" and not queryChildren:
-			logger.debug("Send registration form to user (jid: %s)" % source)
-			# Something is really messed up down here
-			form = utils.buildDataForm(fields = [
-				# Auth page input
-				{"var": "link", "type": "text-single", "label": _("Autorization page"),
-					"desc": ("If you won't get access-token automatically, please, follow authorization link and authorize app,\n"\
-						   "and then paste url to password field."), 
-				"value": URL_ACCEPT_APP},
-				# Phone input
-				{"var": "phone", "type": "text-single", "label": _("Phone number"), "desc": _("Enter phone number in format +71234567890"), "value": "+"},
-				# Password checkbox
-				{"var": "use_password", "type": "boolean", "label": _("Get access-token automatically"), "desc": _("Tries to get access-token automatically. (NOT recommended, password required!)")},
-				# Password input
-				{"var": "password", "type": "text-private", "label": _("Password/Access-token"), "desc": _("Type password, access-token or url (recommended)")}],
-			data = [_("Type data in fields")])
+			logger.debug("Send registration form to user (jid: %s)", source)
+			form = utils.buildDataForm(fields=forms.Forms.getComlicatedForm(), data=[_("Fill the fields below")])
 			result.setQueryPayload([form])
 
 		elif iType == "set" and queryChildren:
diff --git a/modules/mod_iq_vcard.py b/modules/mod_iq_vcard.py
index f62bbc1..b0f5b7d 100644
--- a/modules/mod_iq_vcard.py
+++ b/modules/mod_iq_vcard.py
@@ -10,6 +10,13 @@ VCARD_SEMAPHORE = threading.Semaphore()
 DESCRIPTION = "VK4XMPP Transport\n© simpleApps, 2013 — 2015."
 GITHUB_URL = "https://github.com/mrDoctorWho/vk4xmpp"
 
+if AdditionalAbout:
+	DESCRIPTION = "%s\n%s" % (DESCRIPTION, AdditionalAbout)
+
+VCARD_FIELDS = {"NICKNAME": IDENTIFIER["name"],
+	"DESC": DESCRIPTION,
+	"PHOTO": URL_VCARD_NO_IMAGE,
+	"URL": GITHUB_URL}
 
 def buildVcard(tags):
 	vCard = xmpp.Node("vCard", {"xmlns": xmpp.NS_VCARD})
@@ -25,7 +32,7 @@ def buildVcard(tags):
 @utils.threaded
 def vcard_handler(cl, iq):
 	# Vcard feature makes transport hang (especially the photo part)
-	# Many clients love to query vcards so much, so the solution was in adding a semaphore and sleep() here
+	# Many clients love to query vcards so much, so the solution was in adding a semaphore here and sleep() at the bottom
 	# This is probably not a good idea, but for now this is the best one
 	with VCARD_SEMAPHORE:
 		jidFrom = iq.getFrom()
@@ -33,17 +40,9 @@ def vcard_handler(cl, iq):
 		source = jidFrom.getStripped()
 		destination = jidTo.getStripped()
 		result = iq.buildReply("result")
-		if AdditionalAbout:
-			desc = "%s\n%s" % (DESCRIPTION, AdditionalAbout)
-		else:
-			desc = DESCRIPTION
 
 		if destination == TransportID:
-			vcard = buildVcard({"NICKNAME": IDENTIFIER["name"],
-								"DESC": desc,
-								"PHOTO": URL_VCARD_NO_IMAGE,
-								"URL": GITHUB_URL
-								})
+			vcard = buildVcard(VCARD_FIELDS)
 			result.setPayload([vcard])
 
 		elif source in Transport:
@@ -51,18 +50,19 @@ def vcard_handler(cl, iq):
 			if user.friends:
 				id = vk2xmpp(destination)
 				args = ["screen_name"]
-				if user.friends.has_key(id):
+				values = VCARD_FIELDS.copy()
+				if id in user.friends.keys():
 					args.append(PhotoSize)
-				json = user.vk.getUserData(id, args)
-				name = json.get("name", str(json))
-				screen_name = json.get("screen_name", str(json))
-				values = {"NICKNAME": screen_name,
-						"FN": name,
-						"URL": "http://vk.com/id%s" % id,
-						"DESC": _("Contact uses VK4XMPP Transport\n%s") % desc
-						}
+				data = user.vk.getUserData(id, args)
+				name = data.get("name", str(data))
+				screen_name = data.get("screen_name")
+				if not user.settings.use_nicknames:
+					screen_name = name
+				values["NICKNAME"] = screen_name
+				values["FN"] = name
+				values["URL"] = "http://vk.com/id%s" % id
 				if id in user.friends.keys():
-					values["PHOTO"] = json.get(PhotoSize) or URL_VCARD_NO_IMAGE
+					values["PHOTO"] = data.get(PhotoSize) or URL_VCARD_NO_IMAGE
 				vCard = buildVcard(values)
 				result.setPayload([vCard])
 			else: