From be2ca1ec8084cc7f5054795b3526d924f5acab3b Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 7 Oct 2009 12:56:55 +0200 Subject: Fix build. - Add -f flag to rm on make clean. - Use old jsmin - the new one seems to be broken. --- tools/js2c.py | 77 +++++++------------- tools/jsmin.py | 219 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 245 insertions(+), 51 deletions(-) mode change 120000 => 100644 tools/jsmin.py (limited to 'tools') diff --git a/tools/js2c.py b/tools/js2c.py index 48abe9c5008..312626101b9 100755 --- a/tools/js2c.py +++ b/tools/js2c.py @@ -45,10 +45,19 @@ def ToCArray(lines): return ", ".join(result) -def RemoveCommentsAndTrailingWhitespace(lines): - lines = re.sub(r'//.*\n', '\n', lines) # end-of-line comments - lines = re.sub(re.compile(r'/\*.*?\*/', re.DOTALL), '', lines) # comments. - lines = re.sub(r'\s+\n+', '\n', lines) # trailing whitespace +def CompressScript(lines, do_jsmin): + # If we're not expecting this code to be user visible, we can run it through + # a more aggressive minifier. + if do_jsmin: + return jsmin.jsmin(lines) + + # Remove stuff from the source that we don't want to appear when + # people print the source code using Function.prototype.toString(). + # Note that we could easily compress the scripts mode but don't + # since we want it to remain readable. + #lines = re.sub('//.*\n', '\n', lines) # end-of-line comments + #lines = re.sub(re.compile(r'/\*.*?\*/', re.DOTALL), '', lines) # comments. + #lines = re.sub('\s+\n+', '\n', lines) # trailing whitespace return lines @@ -87,22 +96,6 @@ def ParseValue(string): return string -EVAL_PATTERN = re.compile(r'\beval\s*\('); -WITH_PATTERN = re.compile(r'\bwith\s*\('); - - -def Validate(lines, file): - lines = RemoveCommentsAndTrailingWhitespace(lines) - # Because of simplified context setup, eval and with is not - # allowed in the natives files. - eval_match = EVAL_PATTERN.search(lines) - if eval_match: - raise ("Eval disallowed in natives: %s" % file) - with_match = WITH_PATTERN.search(lines) - if with_match: - raise ("With statements disallowed in natives: %s" % file) - - def ExpandConstants(lines, constants): for key, value in constants.items(): lines = lines.replace(key, str(value)) @@ -162,9 +155,9 @@ class PythonMacro: args.append(mapping[arg]) return str(self.fun(*args)) -CONST_PATTERN = re.compile(r'^const\s+([a-zA-Z0-9_]+)\s*=\s*([^;]*);$') -MACRO_PATTERN = re.compile(r'^macro\s+([a-zA-Z0-9_]+)\s*\(([^)]*)\)\s*=\s*([^;]*);$') -PYTHON_MACRO_PATTERN = re.compile(r'^python\s+macro\s+([a-zA-Z0-9_]+)\s*\(([^)]*)\)\s*=\s*([^;]*);$') +CONST_PATTERN = re.compile('^const\s+([a-zA-Z0-9_]+)\s*=\s*([^;]*);$') +MACRO_PATTERN = re.compile('^macro\s+([a-zA-Z0-9_]+)\s*\(([^)]*)\)\s*=\s*([^;]*);$') +PYTHON_MACRO_PATTERN = re.compile('^python\s+macro\s+([a-zA-Z0-9_]+)\s*\(([^)]*)\)\s*=\s*([^;]*);$') def ReadMacros(lines): constants = { } @@ -212,7 +205,7 @@ namespace node { SOURCE_DECLARATION = """\ - static const char %(id)s[] = { %(data)s }; + static const char native_%(id)s[] = { %(data)s }; """ @@ -230,7 +223,7 @@ GET_DELAY_SCRIPT_NAME_CASE = """\ if (index == %(i)i) return Vector("%(name)s", %(length)i); """ -def JS2C(source, target, env): +def JS2C(source, target): ids = [] delay_ids = [] modules = [] @@ -245,20 +238,16 @@ def JS2C(source, target, env): # Build source code lines source_lines = [ ] - - minifier = jsmin.JavaScriptMinifier() - source_lines_empty = [] - for module in modules: - filename = str(module) - delay = filename.endswith('-delay.js') - lines = ReadFile(filename) + for s in modules: + delay = str(s).endswith('-delay.js') + lines = ReadFile(str(s)) + do_jsmin = lines.find('// jsminify this file, js2c: jsmin') != -1 lines = ExpandConstants(lines, consts) lines = ExpandMacros(lines, macros) - Validate(lines, filename) - lines = minifier.JSMinify(lines) + lines = CompressScript(lines, do_jsmin) data = ToCArray(lines) - id = (os.path.split(filename)[1])[:-3] + id = (os.path.split(str(s))[1])[:-3] if delay: id = id[:-6] if delay: delay_ids.append((id, len(lines))) @@ -266,7 +255,7 @@ def JS2C(source, target, env): ids.append((id, len(lines))) source_lines.append(SOURCE_DECLARATION % { 'id': id, 'data': data }) source_lines_empty.append(SOURCE_DECLARATION % { 'id': id, 'data': 0 }) - + # Build delay support functions get_index_cases = [ ] get_script_source_cases = [ ] @@ -311,8 +300,7 @@ def JS2C(source, target, env): 'source_lines': "\n".join(source_lines), 'get_index_cases': "".join(get_index_cases), 'get_script_source_cases': "".join(get_script_source_cases), - 'get_script_name_cases': "".join(get_script_name_cases), - 'type': env['TYPE'] + 'get_script_name_cases': "".join(get_script_name_cases) }) output.close() @@ -324,17 +312,6 @@ def JS2C(source, target, env): 'source_lines': "\n".join(source_lines_empty), 'get_index_cases': "".join(get_index_cases), 'get_script_source_cases': "".join(get_script_source_cases), - 'get_script_name_cases': "".join(get_script_name_cases), - 'type': env['TYPE'] + 'get_script_name_cases': "".join(get_script_name_cases) }) output.close() - -def main(): - natives = sys.argv[1] - natives_empty = sys.argv[2] - type = sys.argv[3] - source_files = sys.argv[4:] - JS2C(source_files, [natives, natives_empty], { 'TYPE': type }) - -if __name__ == "__main__": - main() diff --git a/tools/jsmin.py b/tools/jsmin.py deleted file mode 120000 index 76e4845a60a..00000000000 --- a/tools/jsmin.py +++ /dev/null @@ -1 +0,0 @@ -../deps/v8/tools/jsmin.py \ No newline at end of file diff --git a/tools/jsmin.py b/tools/jsmin.py new file mode 100644 index 00000000000..ae7581413a9 --- /dev/null +++ b/tools/jsmin.py @@ -0,0 +1,218 @@ +#!/usr/bin/python + +# This code is original from jsmin by Douglas Crockford, it was translated to +# Python by Baruch Even. The original code had the following copyright and +# license. +# +# /* jsmin.c +# 2007-05-22 +# +# Copyright (c) 2002 Douglas Crockford (www.crockford.com) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +# of the Software, and to permit persons to whom the Software is furnished to do +# so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# The Software shall be used for Good, not Evil. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# */ + +from StringIO import StringIO + +def jsmin(js): + ins = StringIO(js) + outs = StringIO() + JavascriptMinify().minify(ins, outs) + str = outs.getvalue() + if len(str) > 0 and str[0] == '\n': + str = str[1:] + return str + +def isAlphanum(c): + """return true if the character is a letter, digit, underscore, + dollar sign, or non-ASCII character. + """ + return ((c >= 'a' and c <= 'z') or (c >= '0' and c <= '9') or + (c >= 'A' and c <= 'Z') or c == '_' or c == '$' or c == '\\' or (c is not None and ord(c) > 126)); + +class UnterminatedComment(Exception): + pass + +class UnterminatedStringLiteral(Exception): + pass + +class UnterminatedRegularExpression(Exception): + pass + +class JavascriptMinify(object): + + def _outA(self): + self.outstream.write(self.theA) + def _outB(self): + self.outstream.write(self.theB) + + def _get(self): + """return the next character from stdin. Watch out for lookahead. If + the character is a control character, translate it to a space or + linefeed. + """ + c = self.theLookahead + self.theLookahead = None + if c == None: + c = self.instream.read(1) + if c >= ' ' or c == '\n': + return c + if c == '': # EOF + return '\000' + if c == '\r': + return '\n' + return ' ' + + def _peek(self): + self.theLookahead = self._get() + return self.theLookahead + + def _next(self): + """get the next character, excluding comments. peek() is used to see + if an unescaped '/' is followed by a '/' or '*'. + """ + c = self._get() + if c == '/' and self.theA != '\\': + p = self._peek() + if p == '/': + c = self._get() + while c > '\n': + c = self._get() + return c + if p == '*': + c = self._get() + while 1: + c = self._get() + if c == '*': + if self._peek() == '/': + self._get() + return ' ' + if c == '\000': + raise UnterminatedComment() + + return c + + def _action(self, action): + """do something! What you do is determined by the argument: + 1 Output A. Copy B to A. Get the next B. + 2 Copy B to A. Get the next B. (Delete A). + 3 Get the next B. (Delete B). + action treats a string as a single character. Wow! + action recognizes a regular expression if it is preceded by ( or , or =. + """ + if action <= 1: + self._outA() + + if action <= 2: + self.theA = self.theB + if self.theA == "'" or self.theA == '"': + while 1: + self._outA() + self.theA = self._get() + if self.theA == self.theB: + break + if self.theA <= '\n': + raise UnterminatedStringLiteral() + if self.theA == '\\': + self._outA() + self.theA = self._get() + + + if action <= 3: + self.theB = self._next() + if self.theB == '/' and (self.theA == '(' or self.theA == ',' or + self.theA == '=' or self.theA == ':' or + self.theA == '[' or self.theA == '?' or + self.theA == '!' or self.theA == '&' or + self.theA == '|' or self.theA == ';' or + self.theA == '{' or self.theA == '}' or + self.theA == '\n'): + self._outA() + self._outB() + while 1: + self.theA = self._get() + if self.theA == '/': + break + elif self.theA == '\\': + self._outA() + self.theA = self._get() + elif self.theA <= '\n': + raise UnterminatedRegularExpression() + self._outA() + self.theB = self._next() + + + def _jsmin(self): + """Copy the input to the output, deleting the characters which are + insignificant to JavaScript. Comments will be removed. Tabs will be + replaced with spaces. Carriage returns will be replaced with linefeeds. + Most spaces and linefeeds will be removed. + """ + self.theA = '\n' + self._action(3) + + while self.theA != '\000': + if self.theA == ' ': + if isAlphanum(self.theB): + self._action(1) + else: + self._action(2) + elif self.theA == '\n': + if self.theB in ['{', '[', '(', '+', '-']: + self._action(1) + elif self.theB == ' ': + self._action(3) + else: + if isAlphanum(self.theB): + self._action(1) + else: + self._action(2) + else: + if self.theB == ' ': + if isAlphanum(self.theA): + self._action(1) + else: + self._action(3) + elif self.theB == '\n': + if self.theA in ['}', ']', ')', '+', '-', '"', '\'']: + self._action(1) + else: + if isAlphanum(self.theA): + self._action(1) + else: + self._action(3) + else: + self._action(1) + + def minify(self, instream, outstream): + self.instream = instream + self.outstream = outstream + self.theA = '\n' + self.theB = None + self.theLookahead = None + + self._jsmin() + self.instream.close() + +if __name__ == '__main__': + import sys + jsm = JavascriptMinify() + jsm.minify(sys.stdin, sys.stdout) -- cgit v1.2.3