From e548e3e1d81e03c94993734de8af1ed7670a6d1c Mon Sep 17 00:00:00 2001 From: Luca Bonavita Date: Sat, 30 Oct 2010 13:25:24 +0000 Subject: == blender file format == Hello, from the bconf 2010 from Jeroen and Luca. Our first combined commit :) Automatically create sdna documentations from Trunk. Usage: blender2.5 -b -P BlendFileDnaExporter_25.py [-- [options]] Options: --dna-keep-blend: doesn't delete the produced blend file DNA export to html --dna-debug: sets the logging level to DEBUG (lots of additional info) --dna-versioned' saves version informations in the html and blend filenames --dna-overwrite-css' overwrite dna.css, useful when modifying css in the script Examples: default: % blender2.5 -b -P BlendFileDnaExporter_25.py with options: % blender2.5 -b -P BlendFileDnaExporter_25.py -- --dna-keep-blend --dna-debug --- doc/blender_file_format/BlendFileDnaExporter_25.py | 477 ++++++++++++ doc/blender_file_format/BlendFileReader.py | 446 +++++++++++ doc/blender_file_format/mystery_of_the_blend.css | 204 +++++ doc/blender_file_format/mystery_of_the_blend.html | 835 +++++++++++++++++++++ 4 files changed, 1962 insertions(+) create mode 100755 doc/blender_file_format/BlendFileDnaExporter_25.py create mode 100644 doc/blender_file_format/BlendFileReader.py create mode 100644 doc/blender_file_format/mystery_of_the_blend.css create mode 100644 doc/blender_file_format/mystery_of_the_blend.html (limited to 'doc/blender_file_format') diff --git a/doc/blender_file_format/BlendFileDnaExporter_25.py b/doc/blender_file_format/BlendFileDnaExporter_25.py new file mode 100755 index 00000000000..77656f43ae5 --- /dev/null +++ b/doc/blender_file_format/BlendFileDnaExporter_25.py @@ -0,0 +1,477 @@ +#! /usr/bin/env python3 + +# ***** BEGIN GPL LICENSE BLOCK ***** +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# ***** END GPL LICENCE BLOCK ***** + +###################################################### +# +# Name: +# dna.py +# +# Description: +# Creates a browsable DNA output to HTML. +# +# Author: +# Jeroen Bakker +# +# Version: +# v0.1 (12-05-2009) - migration of original source code to python. +# Added code to support blender 2.5 branch +# v0.2 (25-05-2009) - integrated with BlendFileReader.py +# +# Input: +# blender build executable +# +# Output: +# dna.html +# dna.css (will only be created when not existing) +# +# Startup: +# ./blender -P BlendFileDnaExporter.py +# +# Process: +# 1: write blend file with SDNA info +# 2: read blend header from blend file +# 3: seek DNA1 file-block +# 4: read dna record from blend file +# 5: close and eventually delete temp blend file +# 6: export dna to html and css +# 7: quit blender +# +###################################################### + +import struct +import sys +import getopt # command line arguments handling +from string import Template # strings completion + + +# logs +import logging +log = logging.getLogger("BlendFileDnaExporter") + +if '--dna-debug' in sys.argv: + logging.basicConfig(level=logging.DEBUG) +else: + logging.basicConfig(level=logging.INFO) + + +class DNACatalogHTML: + ''' + DNACatalog is a catalog of all information in the DNA1 file-block + ''' + + def __init__(self, catalog, bpy_module = None): + self.Catalog = catalog + self.bpy = bpy_module + + def WriteToHTML(self, handle): + + dna_html_template = """ + + + + + + The mystery of the blend + + +
+ Blender ${version}
+ Internal SDNA structures +
+ Architecture: ${bitness} ${endianness}
+ Build revision: ${revision}
+ File format reference: The mystery of the blend by Jeroen Bakker
+

Index of blender structures

+ + ${structs_content} + + """ + + header = self.Catalog.Header + bpy = self.bpy + + # ${version} and ${revision} + if bpy: + version = '.'.join(map(str, bpy.app.version)) + revision = bpy.app.build_revision[:-1] + else: + version = str(header.Version) + revision = 'Unknown' + + # ${bitness} + if header.PointerSize == 8: + bitness = '64 bit' + else: + bitness = '32 bit' + + # ${endianness} + if header.LittleEndianness: + endianess= 'Little endianness' + else: + endianess= 'Big endianness' + + # ${structs_list} + log.debug("Creating structs index") + structs_list = '' + list_item = '
  • ({0}) {1}
    • \n' + structureIndex = 0 + for structure in self.Catalog.Structs: + structs_list += list_item.format(structureIndex, structure.Type.Name) + structureIndex+=1 + + # ${structs_content} + log.debug("Creating structs content") + structs_content = '' + for structure in self.Catalog.Structs: + log.debug(structure.Type.Name) + structs_content += self.Structure(structure) + + d = dict( + version = version, + revision = revision, + bitness = bitness, + endianness = endianess, + structs_list = structs_list, + structs_content = structs_content + ) + + dna_html = Template(dna_html_template).substitute(d) + dna_html = self.format(dna_html) + handle.write(dna_html) + + def Structure(self, structure): + struct_table_template = """ + + + + + + + + + + + + + + ${fields} + +
    ${struct_name}
    referencestructuretypenameoffsetsize
    +
    +
    """ + + d = dict( + struct_name = structure.Type.Name, + fields = self.StructureFields(structure, None, 0), + size = str(structure.Type.Size) + ) + + struct_table = Template(struct_table_template).substitute(d) + return struct_table + + def StructureFields(self, structure, parentReference, offset): + fields = '' + for field in structure.Fields: + fields += self.StructureField(field, structure, parentReference, offset) + offset += field.Size(self.Catalog.Header) + return fields + + def StructureField(self, field, structure, parentReference, offset): + structure_field_template = """ + + ${reference} + ${struct} + ${type} + ${name} + ${offset} + ${size} + """ + + if field.Type.Structure == None or field.Name.IsPointer(): + + # ${reference} + reference = field.Name.AsReference(parentReference) + + # ${struct} + if parentReference != None: + struct = '{0}'.format(structure.Type.Name) + else: + struct = structure.Type.Name + + # ${type} + type = field.Type.Name + + # ${name} + name = field.Name.Name + + # ${offset} + # offset already set + + # ${size} + size = field.Size(self.Catalog.Header) + + d = dict( + reference = reference, + struct = struct, + type = type, + name = name, + offset = offset, + size = size + ) + + structure_field = Template(structure_field_template).substitute(d) + + elif field.Type.Structure != None: + reference = field.Name.AsReference(parentReference) + structure_field = self.StructureFields(field.Type.Structure, reference, offset) + + return structure_field + + def indent(self, input, dent, startswith = ''): + output = '' + if dent < 0: + for line in input.split('\n'): + dent = abs(dent) + output += line[dent:] + '\n' # unindent of a desired amount + elif dent == 0: + for line in input.split('\n'): + output += line.lstrip() + '\n' # remove indentation completely + elif dent > 0: + for line in input.split('\n'): + output += ' '* dent + line + '\n' + return output + + def format(self, input): + diff = { + '\n' :'', + '\n' :'', + '' :' ', + '\n' :'', + '\n' :'', + '\n' :'' + } + output = self.indent(input, 0) + for key, value in diff.items(): + output = output.replace(key, value) + return output + + def WriteToCSS(self, handle): + ''' + Write the Cascading stylesheet template to the handle + It is expected that the handle is a Filehandle + ''' + css = """ + @CHARSET "ISO-8859-1"; + + body { + font-family: verdana; + font-size: small; + } + + div.title { + font-size: large; + text-align: center; + } + + h1 { + page-break-before: always; + } + + h1, h2 { + background-color: #D3D3D3; + color:#404040; + margin-right: 3%; + padding-left: 40px; + } + + h1:hover{ + background-color: #EBEBEB; + } + + h3 { + padding-left: 40px; + } + + table { + border-width: 1px; + border-style: solid; + border-color: #000000; + border-collapse: collapse; + width: 94%; + margin: 20px 3% 10px; + } + + caption { + margin-bottom: 5px; + } + + th { + background-color: #000000; + color:#ffffff; + padding-left:5px; + padding-right:5px; + } + + tr { + } + + td { + border-width: 1px; + border-style: solid; + border-color: #a0a0a0; + padding-left:5px; + padding-right:5px; + } + + label { + float:right; + margin-right: 3%; + } + + ul.multicolumn { + list-style:none; + float:left; + padding-right:0px; + margin-right:0px; + } + + li.multicolumn { + float:left; + width:200px; + margin-right:0px; + } + + a { + color:#a000a0; + text-decoration:none; + } + + a:hover { + color:#a000a0; + text-decoration:underline; + } + """ + + css = self.indent(css, 0) + + handle.write(css) + + +def usage(): + print("\nUsage: \n\tblender2.5 -b -P BlendFileDnaExporter_25.py [-- [options]]") + print("Options:") + print("\t--dna-keep-blend: doesn't delete the produced blend file DNA export to html") + print("\t--dna-debug: sets the logging level to DEBUG (lots of additional info)") + print("\t--dna-versioned' saves version informations in the html and blend filenames") + print("\t--dna-overwrite-css' overwrite dna.css, useful when modifying css in the script") + print("Examples:") + print("\tdefault: % blender2.5 -b -P BlendFileDnaExporter_25.py") + print("\twith options: % blender2.5 -b -P BlendFileDnaExporter_25.py -- --dna-keep-blend --dna-debug\n") + + +###################################################### +# Main +###################################################### + +def main(): + + import os, os.path + + try: + bpy = __import__('bpy') + + # Files + if '--dna-versioned' in sys.argv: + blender_version = '_'.join(map(str, bpy.app.version)) + filename = 'dna-{0}-{1}_endian-{2}-r{3}'.format(sys.arch, sys.byteorder, blender_version, bpy.app.build_revision[2:-1]) + else: + filename = 'dna' + dir = os.path.dirname(__file__) + Path_Blend = os.path.join(dir, filename + '.blend') # temporary blend file + Path_HTML = os.path.join(dir, filename + '.html') # output html file + Path_CSS = os.path.join(dir, 'dna.css') # output css file + + # create a blend file for dna parsing + if not os.path.exists(Path_Blend): + log.info("1: write temp blend file with SDNA info") + log.info(" saving to: " + Path_Blend) + try: + bpy.ops.wm.save_as_mainfile(filepath = Path_Blend, copy = True, compress = False) + except: + log.error("Filename {0} does not exist and can't be created... quitting".format(Path_Blend)) + return + else: + log.info("1: found blend file with SDNA info") + log.info(" " + Path_Blend) + + # read blend header from blend file + log.info("2: read file:") + + if not dir in sys.path: + sys.path.append(dir) + import BlendFileReader + + handle = BlendFileReader.openBlendFile(Path_Blend) + blendfile = BlendFileReader.BlendFile(handle) + catalog = DNACatalogHTML(blendfile.Catalog, bpy) + + # close temp file + handle.close() + + # deleting or not? + if '--dna-keep-blend' in sys.argv: + # keep the blend, useful for studying hexdumps + log.info("5: closing blend file:") + log.info(" {0}".format(Path_Blend)) + else: + # delete the blend + log.info("5: close and delete temp blend:") + log.info(" {0}".format(Path_Blend)) + os.remove(Path_Blend) + + # export dna to xhtml + log.info("6: export sdna to xhtml file") + handleHTML = open(Path_HTML, "w") + catalog.WriteToHTML(handleHTML) + handleHTML.close() + + # only write the css when doesn't exist or at explicit request + if not os.path.exists(Path_CSS) or '--dna-overwrite-css' in sys.argv: + handleCSS = open(Path_CSS, "w") + catalog.WriteToCSS(handleCSS) + handleCSS.close() + + # quit blender + if not bpy.app.background: + log.info("7: quit blender") + bpy.ops.wm.exit_blender() + + except ImportError: + log.warning(" skipping, not running in Blender") + usage() + sys.exit(2) + + +if __name__ == '__main__': + main() diff --git a/doc/blender_file_format/BlendFileReader.py b/doc/blender_file_format/BlendFileReader.py new file mode 100644 index 00000000000..7003af10ac7 --- /dev/null +++ b/doc/blender_file_format/BlendFileReader.py @@ -0,0 +1,446 @@ +#! /usr/bin/env python3 + +# ***** BEGIN GPL LICENSE BLOCK ***** +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# ***** END GPL LICENCE BLOCK ***** + +###################################################### +# Importing modules +###################################################### + +import os +import struct +import gzip +import tempfile + +import logging +log = logging.getLogger("BlendFileReader") + +###################################################### +# module global routines +###################################################### + +def ReadString(handle, length): + ''' + ReadString reads a String of given length or a zero terminating String + from a file handle + ''' + if length != 0: + return handle.read(length).decode() + else: + # length == 0 means we want a zero terminating string + result = "" + s = ReadString(handle, 1) + while s!="\0": + result += s + s = ReadString(handle, 1) + return result + + +def Read(type, handle, fileheader): + ''' + Reads the chosen type from a file handle + ''' + def unpacked_bytes(type_char, size): + return struct.unpack(fileheader.StructPre + type_char, handle.read(size))[0] + + if type == 'ushort': + return unpacked_bytes("H", 2) # unsigned short + elif type == 'short': + return unpacked_bytes("h", 2) # short + elif type == 'uint': + return unpacked_bytes("I", 4) # unsigned int + elif type == 'int': + return unpacked_bytes("i", 4) # int + elif type == 'float': + return unpacked_bytes("f", 4) # float + elif type == 'ulong': + return unpacked_bytes("Q", 8) # unsigned long + elif type == 'pointer': + # The pointersize is given by the header (BlendFileHeader). + if fileheader.PointerSize == 4: + return Read('uint', handle, fileheader) + if fileheader.PointerSize == 8: + return Read('ulong', handle, fileheader) + + +def openBlendFile(filename): + ''' + Open a filename, determine if the file is compressed and returns a handle + ''' + handle = open(filename, 'rb') + magic = ReadString(handle, 7) + if magic in ("BLENDER", "BULLETf"): + log.debug("normal blendfile detected") + handle.seek(0, os.SEEK_SET) + return handle + else: + log.debug("gzip blendfile detected?") + handle.close() + log.debug("decompressing started") + fs = gzip.open(filename, "rb") + handle = tempfile.TemporaryFile() + data = fs.read(1024*1024) + while data: + handle.write(data) + data = fs.read(1024*1024) + log.debug("decompressing finished") + fs.close() + log.debug("resetting decompressed file") + handle.seek(0, os.SEEK_SET) + return handle + + +def Align(handle): + ''' + Aligns the filehandle on 4 bytes + ''' + offset = handle.tell() + trim = offset % 4 + if trim != 0: + handle.seek(4-trim, os.SEEK_CUR) + + +###################################################### +# module classes +###################################################### + +class BlendFile: + ''' + Reads a blendfile and store the header, all the fileblocks, and catalogue + structs foound in the DNA fileblock + + - BlendFile.Header (BlendFileHeader instance) + - BlendFile.Blocks (list of BlendFileBlock instances) + - BlendFile.Catalog (DNACatalog instance) + ''' + + def __init__(self, handle): + log.debug("initializing reading blend-file") + self.Header = BlendFileHeader(handle) + self.Blocks = [] + fileblock = BlendFileBlock(handle, self) + found_dna_block = False + while not found_dna_block: + if fileblock.Header.Code in ("DNA1", "SDNA"): + self.Catalog = DNACatalog(self.Header, handle) + found_dna_block = True + else: + fileblock.Header.skip(handle) + + self.Blocks.append(fileblock) + fileblock = BlendFileBlock(handle, self) + + # appending last fileblock, "ENDB" + self.Blocks.append(fileblock) + + # seems unused? + """ + def FindBlendFileBlocksWithCode(self, code): + #result = [] + #for block in self.Blocks: + #if block.Header.Code.startswith(code) or block.Header.Code.endswith(code): + #result.append(block) + #return result + """ + + +class BlendFileHeader: + ''' + BlendFileHeader allocates the first 12 bytes of a blend file. + It contains information about the hardware architecture. + Header example: BLENDER_v254 + + BlendFileHeader.Magic (str) + BlendFileHeader.PointerSize (int) + BlendFileHeader.LittleEndianness (bool) + BlendFileHeader.StructPre (str) see http://docs.python.org/py3k/library/struct.html#byte-order-size-and-alignment + BlendFileHeader.Version (int) + ''' + + def __init__(self, handle): + log.debug("reading blend-file-header") + + self.Magic = ReadString(handle, 7) + log.debug(self.Magic) + + pointersize = ReadString(handle, 1) + log.debug(pointersize) + if pointersize == "-": + self.PointerSize = 8 + if pointersize == "_": + self.PointerSize = 4 + + endianness = ReadString(handle, 1) + log.debug(endianness) + if endianness == "v": + self.LittleEndianness = True + self.StructPre = "<" + if endianness == "V": + self.LittleEndianness = False + self.StructPre = ">" + + version = ReadString(handle, 3) + log.debug(version) + self.Version = int(version) + + log.debug("{0} {1} {2} {3}".format(self.Magic, self.PointerSize, self.LittleEndianness, version)) + + +class BlendFileBlock: + ''' + BlendFileBlock.File (BlendFile) + BlendFileBlock.Header (FileBlockHeader) + ''' + + def __init__(self, handle, blendfile): + self.File = blendfile + self.Header = FileBlockHeader(handle, blendfile.Header) + + def Get(self, handle, path): + log.debug("find dna structure") + dnaIndex = self.Header.SDNAIndex + dnaStruct = self.File.Catalog.Structs[dnaIndex] + log.debug("found " + dnaStruct.Type.Name) + handle.seek(self.Header.FileOffset, os.SEEK_SET) + return dnaStruct.GetField(self.File.Header, handle, path) + + +class FileBlockHeader: + ''' + FileBlockHeader contains the information in a file-block-header. + The class is needed for searching to the correct file-block (containing Code: DNA1) + + Code (str) + Size (int) + OldAddress (pointer) + SDNAIndex (int) + Count (int) + FileOffset (= file pointer of datablock) + ''' + + def __init__(self, handle, fileheader): + self.Code = ReadString(handle, 4).strip() + if self.Code != "ENDB": + self.Size = Read('uint', handle, fileheader) + self.OldAddress = Read('pointer', handle, fileheader) + self.SDNAIndex = Read('uint', handle, fileheader) + self.Count = Read('uint', handle, fileheader) + self.FileOffset = handle.tell() + else: + self.Size = Read('uint', handle, fileheader) + self.OldAddress = 0 + self.SDNAIndex = 0 + self.Count = 0 + self.FileOffset = handle.tell() + #self.Code += ' ' * (4 - len(self.Code)) + log.debug("found blend-file-block-fileheader {0} {1}".format(self.Code, self.FileOffset)) + + def skip(self, handle): + handle.read(self.Size) + + +class DNACatalog: + ''' + DNACatalog is a catalog of all information in the DNA1 file-block + + Header = None + Names = None + Types = None + Structs = None + ''' + + def __init__(self, fileheader, handle): + log.debug("building DNA catalog") + self.Names=[] + self.Types=[] + self.Structs=[] + self.Header = fileheader + + SDNA = ReadString(handle, 4) + + # names + NAME = ReadString(handle, 4) + numberOfNames = Read('uint', handle, fileheader) + log.debug("building #{0} names".format(numberOfNames)) + for i in range(numberOfNames): + name = ReadString(handle,0) + self.Names.append(DNAName(name)) + Align(handle) + + # types + TYPE = ReadString(handle, 4) + numberOfTypes = Read('uint', handle, fileheader) + log.debug("building #{0} types".format(numberOfTypes)) + for i in range(numberOfTypes): + type = ReadString(handle,0) + self.Types.append(DNAType(type)) + Align(handle) + + # type lengths + TLEN = ReadString(handle, 4) + log.debug("building #{0} type-lengths".format(numberOfTypes)) + for i in range(numberOfTypes): + length = Read('ushort', handle, fileheader) + self.Types[i].Size = length + Align(handle) + + # structs + STRC = ReadString(handle, 4) + numberOfStructures = Read('uint', handle, fileheader) + log.debug("building #{0} structures".format(numberOfStructures)) + for structureIndex in range(numberOfStructures): + type = Read('ushort', handle, fileheader) + Type = self.Types[type] + structure = DNAStructure(Type) + self.Structs.append(structure) + + numberOfFields = Read('ushort', handle, fileheader) + for fieldIndex in range(numberOfFields): + fTypeIndex = Read('ushort', handle, fileheader) + fNameIndex = Read('ushort', handle, fileheader) + fType = self.Types[fTypeIndex] + fName = self.Names[fNameIndex] + structure.Fields.append(DNAField(fType, fName)) + + +class DNAName: + ''' + DNAName is a C-type name stored in the DNA. + + Name = str + ''' + + def __init__(self, name): + self.Name = name + + def AsReference(self, parent): + if parent == None: + result = "" + else: + result = parent+"." + + result = result + self.ShortName() + return result + + def ShortName(self): + result = self.Name; + result = result.replace("*", "") + result = result.replace("(", "") + result = result.replace(")", "") + Index = result.find("[") + if Index != -1: + result = result[0:Index] + return result + + def IsPointer(self): + return self.Name.find("*")>-1 + + def IsMethodPointer(self): + return self.Name.find("(*")>-1 + + def ArraySize(self): + result = 1 + Temp = self.Name + Index = Temp.find("[") + + while Index != -1: + Index2 = Temp.find("]") + result*=int(Temp[Index+1:Index2]) + Temp = Temp[Index2+1:] + Index = Temp.find("[") + + return result + + +class DNAType: + ''' + DNAType is a C-type stored in the DNA + + Name = str + Size = int + Structure = DNAStructure + ''' + + def __init__(self, aName): + self.Name = aName + self.Structure=None + + +class DNAStructure: + ''' + DNAType is a C-type structure stored in the DNA + + Type = DNAType + Fields = [DNAField] + ''' + + def __init__(self, aType): + self.Type = aType + self.Type.Structure = self + self.Fields=[] + + def GetField(self, header, handle, path): + splitted = path.partition(".") + name = splitted[0] + rest = splitted[2] + offset = 0; + for field in self.Fields: + if field.Name.ShortName() == name: + log.debug("found "+name+"@"+str(offset)) + handle.seek(offset, os.SEEK_CUR) + return field.DecodeField(header, handle, rest) + else: + offset += field.Size(header) + + log.debug("error did not find "+path) + return None + + +class DNAField: + ''' + DNAField is a coupled DNAType and DNAName. + + Type = DNAType + Name = DNAName + ''' + + def __init__(self, aType, aName): + self.Type = aType + self.Name = aName + + def Size(self, header): + if self.Name.IsPointer() or self.Name.IsMethodPointer(): + return header.PointerSize*self.Name.ArraySize() + else: + return self.Type.Size*self.Name.ArraySize() + + def DecodeField(self, header, handle, path): + if path == "": + if self.Name.IsPointer(): + return Read('pointer', handle, header) + if self.Type.Name=="int": + return Read('int', handle, header) + if self.Type.Name=="short": + return Read('short', handle, header) + if self.Type.Name=="float": + return Read('float', handle, header) + if self.Type.Name=="char": + return ReadString(handle, self.Name.ArraySize()) + else: + return self.Type.Structure.GetField(header, handle, path) + diff --git a/doc/blender_file_format/mystery_of_the_blend.css b/doc/blender_file_format/mystery_of_the_blend.css new file mode 100644 index 00000000000..df287b54a06 --- /dev/null +++ b/doc/blender_file_format/mystery_of_the_blend.css @@ -0,0 +1,204 @@ +@CHARSET "ISO-8859-1"; + +table { + border-width: 1px; + border-style: solid; + border-color: #000000; + border-collapse: collapse; + width: 94%; + margin: 10px 3%; +} + +DIV.title { + font-size: 30px; + font-weight: bold; + text-align: center +} + +DIV.subtitle { + font-size: large; + text-align: center +} + +DIV.contact { + margin:30px 3%; +} + +@media print { + DIV.contact { + margin-top: 300px; + } + DIV.title { + margin-top: 400px; + } +} + +label { + font-weight: bold; + width: 100px; + float: left; +} + +label:after { + content: ":"; +} + +TH { + background-color: #000000; + color: #ffffff; + padding-left: 5px; + padding-right: 5px; +} + +TR { +} + +TD { + border-width: 1px; + border-style: solid; + border-color: #a0a0a0; + padding-left: 5px; + padding-right: 5px; +} + +BODY { + font-family: verdana; + font-size: small; +} + +H1 { + page-break-before: always; +} + +H1, H2, H3, H4 { + margin-top: 30px; + margin-right: 3%; + padding: 3px 3%; + color: #404040; + cursor: pointer; +} + +H1, H2 { + background-color: #D3D3D3; +} + +H3, H4 { + padding-top: 5px; + padding-bottom: 5px; +} + +H1:hover, H2:hover, H3:hover, H4:hover { + background-color: #EBEBEB; +} + +CODE.evidence { + font-size:larger +} + +CODE.block { + color: #000000; + background-color: #DDDC75; + margin: 10px 0; + padding: 5px; + border-width: 1px; + border-style: dotted; + border-color: #000000; + white-space: pre; + display: block; + font-size: 2 em; +} + +ul { + margin: 10px 3%; +} + +li { + margin: 0 -15px; +} + +ul.multicolumn { + list-style: none; + float: left; + padding-right: 0px; + margin-right: 0px; +} + +li.multicolumn { + float: left; + width: 200px; + margin-right: 0px; +} + +@media screen { + p { + margin: 10px 3%; + line-height: 130%; + } +} + +span.fade { + color: gray; +} + +span.header { + color: green; +} + +span.header-greyed { + color: #4CBE4B; +} + +span.data { + color: blue; +} + +span.data-greyed { + color: #5D99C4; +} + +span.descr { + color: red; +} + +div.box { + margin: 15px 3%; + border-style: dotted; + border-width: 1px; +} + +div.box-solid { + margin: 15px 3%; + border-style: solid; + border-width: 1px; +} + +p.box-title { + font-style: italic; + font-size: 110%; + cursor: pointer; +} + +p.box-title:hover { + background-color: #EBEBEB; +} + +p.code { + font-family: "Courier New", Courier, monospace; +} + +a { + color: #a000a0; + text-decoration: none; +} + +a:hover { + color: #a000a0; + text-decoration: underline; +} + +td.skip { + color: #808080; + padding-top: 10px; + padding-bottom: 10px; + text-align: center; +} diff --git a/doc/blender_file_format/mystery_of_the_blend.html b/doc/blender_file_format/mystery_of_the_blend.html new file mode 100644 index 00000000000..b34493ffa3e --- /dev/null +++ b/doc/blender_file_format/mystery_of_the_blend.html @@ -0,0 +1,835 @@ + + + + + + The mystery of the blend + + + +
    The mystery of the blend
    +
    The blender file-format explained
    +
    + Jeroen Bakker
    +     j.bakker@atmind.nl
    + http://www.atmind.nl/blender
    + 06-10-2010
    +
    + +

    Introduction

     + + +

    In this article I will describe the + blend-file-format with a request to tool-makers to support blend-file. + +

    +

    First I'll describe how Blender works with blend-files. You'll notice + why the blend-file-format is not that well documented, as from +Blender's perspective this is not needed. +We look at the global file-structure of a blend-file (the file-header +and file-blocks). +After this is explained, we go deeper to the core of the blend-file, the + DNA-structures. They hold the blue-prints of the blend-file and the key + asset of understanding blend-files. +When that's done we can use these DNA-structures to read information +from elsewhere in the blend-file. + +

    +

    +In this article we'll be using the default blend-file from Blender 2.54, + with the goal to read the output resolution from the Scene. +The article is written to be programming language independent and I've +setup a web-site for support. +

    + + +

    Loading and saving in Blender

    +     + +

    +Loading and saving in Blender is very fast and Blender is known to +have excellent downward and upward compatibility. Ton Roosendaal +demonstrated that in December 2008 by loading a 1.0 blend-file using +Blender 2.48a [ref: http://www.blendernation.com/2008/12/01/blender-dna-rna-and-backward-compatibility/]. +

    + +

    +Saving complex scenes in Blender is done within seconds. Blender +achieves this by saving data in memory to disk without any +transformations or translations. Blender only adds file-block-headers to + this data. A file-block-header contains clues on how to interpret the +data. After the data, all internally Blender structures are stored. +These structures will act as blue-prints when Blender loads the file. +Blend-files can be different when stored on different hardware platforms + or Blender releases. There is no effort taken to make blend-files +binary the same. Blender creates the blend-files in this manner since +release 1.0. Backward and upwards compatibility is not implemented when +saving the file, this is done during loading. +

    + +

    +When Blender loads a blend-file, the DNA-structures are read first. +Blender creates a catalog of these DNA-structures. Blender uses this +catalog together with the data in the file, the internal Blender +structures of the Blender release you're using and a lot of +transformation and translation logic to implement the backward and +upward compatibility. In the source code of blender there is actually +logic which can transform and translate every structure used by a +Blender release to the one of the release you're using [ref: http://download.blender.org/source/blender-2.48a.tar.gz + blender/blenloader/intern/readfile.c lines +4946-7960]. The more difference between releases the more logic is +executed. +

    + +

    +The blend-file-format is not well documented, as it does not differ from + internally used structures and the file can really explain itself. +

    + + +

    Global file-structure

    +     + +

    +This section explains how the global file-structure can be read. +

    + + + + +
    + +

    File.blend

    + +

    File-header

    + +

    File-block

    +

    Header

    +

    Data

    +
    + +

    File-block

    +

    File-block

    + +

    File-block 'Structure DNA'

    +

    Header ('DNA1')

    +
    +

    Data ('SDNA')

    +
    +

    Names ('NAME')

    +
    +
    +

    Types ('TYPE')

    +
    +
    +

    Lengths ('TLEN')

    +
    +
    +

    Structures ('STRC')

    +
    +
    +
    + +

    File-Block 'ENDB'

    + +
    + + +

    File-Header

    +     + +

    +The first 12 bytes of every blend-file is the file-header. The +file-header has information on Blender (version-number) and the PC the +blend-file was saved on (pointer-size and endianness). This is required +as all data inside the blend-file is ordered in that way, because no +translation or transformation is done during saving. +The next table describes the information in the file-header. +

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    File-header
    referencestructuretypeoffsetsize
    identifierchar[7]File identifier (always 'BLENDER')07
    pointer-sizecharSize of a pointer; all pointers in the file are stored in this format. '_' means 4 bytes or 32 bit and '-' means 8 bytes or 64 bits.71
    endiannesscharType of byte ordering used; 'v' means little endian and 'V' means big endian.81
    version-numberchar[3]Version of Blender the file was created in; '254' means version 2.5493
    + +

    +Endianness addresses the way values are ordered in a sequence of bytes(see the example below): +

    + + + +

    +Nowadays, little-endian is the most commonly used. +

    + + +
    +

    +Endianess Example +

    +

    +Writing the integer 0x4A3B2C1Dh, will be ordered: +

    +

    +
    + +

    +Blender supports little-endian and big-endian.
    +This means that when the endianness +is different between the blend-file and the PC your using, Blender changes it to the byte ordering +of your PC. +

    + + +
    +

    +File-header Example +

    + +

    +This hex-dump describes a file-header created with blender 2.54.0 on little-endian hardware with a 32 bits pointer length. + pointer-size version-number + | | +0000 0000: [42 4C 45 4E 44 45 52] [5F] [76] [32 35 34] BLENDER_v254 + | | + identifier endianness +

    +
    + +

    File-blocks

     + +

    +File-blocks contain a "file-block header" and "file-block data". +

    + +

    File-block headers

     + +

    +The file-block-header describes: +

    + + + +

    +As we can see below, depending on the pointer-size stored in the file-header, a file-block-header +can be 20 or 24 bytes long, hence it is always aligned at 4 bytes. +

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    File-block-header
    referencestructuretypeoffsetsize
    codechar[4]File-block identifier04
    sizeintegerTotal length of the data after the file-block-header44
    old memory addressvoid*Memory address the structure was located when written to disk8pointer-size (4/8)
    SDNA indexintegerIndex of the SDNA structure8+pointer-size4
    countintegerNumber of structure located in this file-block12+pointer-size4
    + +

    +The above table describes how a file-block-header is structured: +

    + + + + +
    +

    +Example +

    +

    +This hex-dump describes a File-block (= File-block header + File-block data) created with blender 2.54 on little-endian hardware with a 32 bits pointer length.
    + file-block + identifier='SC' data size=1404 old pointer SDNA index=150 + | | | | +0000 4420: [53 43 00 00] [7C 05 00 00] [68 34 FB 0B] [96 00 00 00] SC.. `... ./.. .... +0000 4430: [01 00 00 00] [xx xx xx xx xx xx xx xx xx xx xx xx .... xxxx xxxx xxxx + | | + count=1 file-block data (next 1404 bytes) + +

    + + + +

    +Before we can interpret the data of this file-block we first have to read the DNA structures in the file. +The section "Structure DNA" will show how to do that. +

    +
    + +

    Structure DNA

     + +

    The DNA1 file-block

     + +

    +Structure DNA is stored in a file-block with code 'DNA1'. It can be just before the 'ENDB' file-block. +

    + +

    +The 'DNA1' file-block contains all internal structures of the Blender release the +file was created in.
    +These structure can be described as C-structures: they can hold fields, arrays and +pointers to other structures, just like a normal C-structure. + +

    +struct SceneRenderLayer { + struct SceneRenderLayer *next, *prev; + char name[32]; + struct Material *mat_override; + struct Group *light_override; + unsigned int lay; + unsigned int lay_zmask; + int layflag; + int pad; + int passflag; + int pass_xor; +}; + +

    + +

    +For example,a blend-file created with Blender 2.54 the 'DNA1' file-block is 57796 bytes long and contains 398 structures. +

    + +

    DNA1 file-block-header

     + +

    +The DNA1 file-block header follows the same rules of any other file-block, see the example below. +

    + + +
    +

    +Example +

    +

    +This hex-dump describes the file-block 'DNA1' header created with blender 2.54.0 on little-endian hardware with a 32 bits pointer length.
    + (file-block + identifier='DNA1') data size=57796 old pointer SDNA index=0 + | | | | +0004 B060 [44 4E 41 31] [C4 E1 00 00] [C8 00 84 0B] [00 00 00 00] DNA1............ +0004 B070 [01 00 00 00] [53 44 4E 41 4E 41 4D 45 CB 0B 00 00 ....SDNANAME.... + | | + count=1 'DNA1' file-block data (next 57796 bytes) + +

    +
    + +

    DNA1 file-block data

     +

    +The next section describes how this information is ordered in the data of the 'DNA1' file-block. +

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Structure of the DNA file-block-data
    repeat conditionnametypelengthdescription
    identifierchar[4]4'SDNA'
    name identifierchar[4]4'NAME'
    #namesinteger4Number of names follows
    for(#names)namechar[]?Zero terminating string of name, also contains pointer and simple array definitions (e.g. '*vertex[3]\0')
    type identifierchar[4]4'TYPE' this field is aligned at 4 bytes
    #typesinteger4Number of types follows
    for(#types)typechar[]?Zero terminating string of type (e.g. 'int\0')
    length identifierchar[4]4'TLEN' this field is aligned at 4 bytes
    for(#types)lengthshort2Length in bytes of type (e.g. 4)
    structure identifierchar[4]4'STRC' this field is aligned at 4 bytes
    #structuresinteger4Number of structures follows
    for(#structures)structure typeshort2Index in types containing the name of the structure
    ..#fieldsshort2Number of fields in this structure
    ..for(#field)field typeshort2Index in type
    for endfor endfield nameshort2Index in name
    + +

    +As you can see, the structures are stored in 4 arrays: names, types, +lengths and structures. Every structure also contains an array of +fields. A field is the combination of a type and a name. From this +information a catalog of all structures can be constructed. +The names are stored as how a C-developer defines them. This means that +the name also defines pointers and arrays. +(When a name starts with '*' it is used as a pointer. when the name +contains for example '[3]' it is used as a array of 3 long.) +In the types you'll find simple-types (like: 'integer', 'char', +'float'), but also complex-types like 'Scene' and 'MetaBall'. +'TLEN' part describes the length of the types. A 'char' is 1 byte, an +'integer' is 4 bytes and a 'Scene' is 1376 bytes long. +

    + +
    +

    +Note +

    +

    +All identifiers, are arrays of 4 chars, hence they are all aligned at 4 bytes. +

    +
    + + +
    +

    +Example +

    +

    +Created with blender 2.54.0 on little-endian hardware with a 32 bits pointer length. +

    + +

    The names array

     +

    +The first names are: *next, *prev, *data, *first, *last, x, y, xmin, xmax, ymin, ymax, *pointer, group, val, val2, type, subtype, flag, name[32], ... + file-block-data identifier='SDNA' array-id='NAME' number of names=3019 + | | | +0004 B070 01 00 00 00 [53 44 4E 41][4E 41 4D 45] [CB 0B 00 00] ....SDNANAME.... +0004 B080 [2A 6E 65 78 74 00][2A 70 72 65 76 00] [2A 64 61 74 *next.*prev.*dat + | | | + '*next\0' '*prev\0' '*dat' + .... + .... (3019 names) + +

    + +
    +

    +Note +

    +

    +While reading the DNA you'll will come across some strange +names like '(*doit)()'. These are method pointers and Blender updates +them to the correct methods. +

    +
    + +

    The types array

     +

    +The first types are: char, uchar, short, ushort, int, long, ulong, float, double, void, Link, LinkData, ListBase, vec2s, vec2f, ... + array-id='TYPE' + | +0005 2440 6F 6C 64 5B 34 5D 5B 34 5D 00 00 00 [54 59 50 45] old[4][4]...TYPE +0005 2450 [C9 01 00 00] [63 68 61 72 00] [75 63 68 61 72 00][73 ....char.uchar.s + | | | | + number of types=457 'char\0' 'uchar\0' 's' + .... + .... (457 types) + +

    + +

    The lengths array

     +

    + char uchar ushort short + array-id length length length length + 'TLEN' 1 1 2 2 +0005 3AA0 45 00 00 00 [54 4C 45 4E] [01 00] [01 00] [02 00] [02 00] E...TLEN........ + .... +0005 3AC0 [08 00] [04 00] [08 00] [10 00] [10 00] [14 00] [4C 00] [34 00] ............L.4. + 8 4 8 + ListBase vec2s vec2f ... etc + length len length + .... + .... (457 lengths, same as number of types) + +

    + +

    The structures array

     +

    + array-id='STRC' + | +0005 3E30 40 00 38 00 60 00 00 00 00 00 00 00 [53 54 52 43] @.8.`.......STRC +0005 3E40 [8E 01 00 00] [0A 00] [02 00] [0A 00] [00 00] [0A 00] [01 00] ................ + 398 10 2 10 0 10 0 + number of index fields index index index index + structures in types in types in names in types in names + ' '----------------' '-----------------' ' + ' field 0 field 1 ' + '--------------------------------------------------------' + structure 0 + .... + .... (398 structures, each one describeing own type, and type/name for each field) + +

    +
    + +

    +The DNA structures inside a Blender 2.48 blend-file can be found at http://www.atmind.nl/blender/blender-sdna.html. + +If we understand the DNA part of the file it is now possible to read +information from other parts file-blocks. The next section will tell us +how. +

    + +

    Reading scene information

     + +

    +Let us look at the file-block header we have seen earlier:
    +

    + +

    +Now note that: +

    +

    + +

    +We can map the Scene structure on the data of the file-blocks. +But before we can do that, we have to flatten the Scene-structure. + +struct Scene { + ID id; // 52 bytes long (ID is different a structure) + AnimData *adt; // 4 bytes long (pointer to an AnimData structure) + Object *camera; // 4 bytes long (pointer to an Object structure) + World *world; // 4 bytes long (pointer to an Object structure) + ... + float cursor[3]; // 12 bytes long (array of 3 floats) + ... +}; + + +The first field in the Scene-structure is of type 'ID' with the name 'id'. +Inside the list of DNA structures there is a structure defined for type 'ID' (structure index 17). + +struct ID { + void *next, *prev; + struct ID *newid; + struct Library *lib; + char name[24]; + short us; + short flag; + int icon_id; + IDProperty *properties; +}; + + +The first field in this structure has type 'void' and name '*next'.
    +Looking in the structure list there is no structure defined for type 'void': it is a simple type and therefore the data should be read. +The name '*next' describes a pointer. +As we see, the first 4 bytes of the data can be mapped to 'id.next'. +

    + +

    +Using this method we'll map a structure to its data. If we want to +read a specific field we know at which offset in the data it is located +and how much space it takes.
    +The next table shows the output of this flattening process for some +parts of the Scene-structure. Not all rows are described in the table as + there is a lot of information in a Scene-structure. +

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Flattened SDNA structure 150: Scene
    referencestructuretypenameoffsetsizedescription
    id.nextIDvoid*next04Refers to the next scene
    id.prevIDvoid*prev44Refers to the previous scene
    id.newidIDID*newid84
    id.libIDLibrary*lib124
    id.nameIDcharname[24]1624'SC'+the name of the scene as displayed in Blender
    id.usIDshortus402
    id.flagIDshortflag422
    id.icon_idIDinticon_id444
    id.propertiesIDIDProperty*properties484
    adtSceneAnimData*adt524
    cameraSceneObject*camera564Pointer to the current camera
    worldSceneWorld*world604Pointer to the current world
    Skipped rows
    r.xschRenderData + shortxsch3822X-resolution of the output when rendered at 100%
    r.yschRenderData + shortysch3842Y-resolution of the output when rendered at 100%
    r.xpartsRenderData + shortxparts3862Number of x-part used by the renderer
    r.ypartsRenderData + shortyparts3882Number of x-part used by the renderer
    Skipped rows
    gpdScenebGPdata*gpd13764
    physics_settings.gravityPhysicsSettings + floatgravity[3]138012
    physics_settings.flagPhysicsSettings + intflag13924
    physics_settings.quick_cache_stepPhysicsSettings + intquick_cache_step13964
    physics_settings.rtPhysicsSettings + intrt14004
    + +

    +We can now read the X and Y resolution of the Scene: +

    +

    + +
    +

    +Note +

    +

    +An array of chars can mean 2 things. The field contains readable +text or it contains an array of flags (not humanly readable). +

    +
    + +
    +

    +Note +

    +

    +A file-block containing a list refers to the DNA structure and has a count larger +than 1. For example Vertexes and Faces are stored in this way. +

    +
    + + + + -- cgit v1.2.3