Lua vs. XML benchmark results

[tp]

I did a quick benchmark test for my project. I'm posting the results in case someone else is interested.

In my case, I am dealing with collision meshes in a custom format, that are basically exported out of blender and then loaded into my engine. I wanted to use a standard-ish format, so I originally went with XML, but something that turned out to be a bug in my engine made me think that they could just as well be parsed as Lua files. I wanted to compare the performance of loading something like this as an XML file or a Lua file.

I used Lua 5.1 and libxml 2.7.2 for the test.

Here is the file I used in XML:

Code: Select all

<?xml version="1.0" encoding="iso-8859-1"?>
<convexhull name="Test">
	<pointcloud>
		<point>
			<x>0.18368</x>
			<y>0.00853</y>
			<z>-0.16557</z>
		</point>
		<point>
			<x>0.23088</x>
			<y>0.00853</y>
			<z>0.29686</z>
		</point>
		<point>
			<x>-0.24137</x>
			<y>0.00853</y>
			<z>0.14946</z>
		</point>
		<point>
			<x>-0.23995</x>
			<y>0.00853</y>
			<z>-0.18997</z>
		</point>
		<point>
			<x>0.19459</x>
			<y>1.42596</y>
			<z>-0.06543</z>
		</point>
		<point>
			<x>0.1688</x>
			<y>1.42596</y>
			<z>0.21986</z>
		</point>
		<point>
			<x>-0.19459</x>
			<y>1.42596</y>
			<z>0.21985</z>
		</point>
		<point>
			<x>-0.19459</x>
			<y>1.42596</y>
			<z>-0.06543</z>
		</point>
		<point>
			<x>0.0</x>
			<y>0.00853</y>
			<z>0.2873</z>
		</point>
		<point>
			<x>0.0</x>
			<y>0.00853</y>
			<z>-0.21985</z>
		</point>
		<point>
			<x>-0.0</x>
			<y>1.69016</y>
			<z>0.21986</z>
		</point>
		<point>
			<x>0.0</x>
			<y>1.75005</y>
			<z>-0.02323</z>
		</point>
		<point>
			<x>0.26395</x>
			<y>0.00853</y>
			<z>0.07559</z>
		</point>
		<point>
			<x>-0.29077</x>
			<y>0.00853</y>
			<z>-0.03154</z>
		</point>
		<point>
			<x>0.26686</x>
			<y>1.44191</y>
			<z>0.06391</z>
		</point>
		<point>
			<x>-0.27044</x>
			<y>1.42596</y>
			<z>0.06391</z>
		</point>
		<point>
			<x>0.0</x>
			<y>0.00853</y>
			<z>0.0</z>
		</point>
		<point>
			<x>0.0</x>
			<y>1.7837</y>
			<z>0.10032</z>
		</point>
		<point>
			<x>0.18368</x>
			<y>1.12971</y>
			<z>0.23308</z>
		</point>
		<point>
			<x>0.18368</x>
			<y>1.12971</y>
			<z>-0.16557</z>
		</point>
		<point>
			<x>-0.23208</x>
			<y>1.12971</y>
			<z>0.21986</z>
		</point>
		<point>
			<x>-0.18368</x>
			<y>1.12971</y>
			<z>-0.16557</z>
		</point>
		<point>
			<x>-0.05467</x>
			<y>1.22532</y>
			<z>0.4318</z>
		</point>
		<point>
			<x>0.0</x>
			<y>1.12971</y>
			<z>-0.21985</z>
		</point>
		<point>
			<x>0.31219</x>
			<y>1.12971</y>
			<z>0.0325</z>
		</point>
		<point>
			<x>-0.30222</x>
			<y>1.12971</y>
			<z>0.0</z>
		</point>
	</pointcloud>
	<placement>
		<translation>
			<x>0.0</x>
			<y>0.15</y>
			<z>0.0</z>
		</translation>
		<orientation>
			<w>1.0</w>
			<x>0.0</x>
			<y>0.0</y>
			<z>0.0</z>
		</orientation>
	</placement>
	<mass>70.0</mass>
</convexhull>

And here is the Lua version:

Code: Select all

return {
	pointcloud = {
		{ 0.18368, 0.00853, -0.16557 },
		{ 0.23088, 0.00853, 0.29686 },
		{ -0.24137, 0.00853, 0.14946 },
		{ -0.23995, 0.00853, -0.18997 },
		{ 0.19459, 1.42596, -0.06543 },
		{ 0.1688, 1.42596, 0.21986 },
		{ -0.19459, 1.42596, 0.21985 },
		{ -0.19459, 1.42596, -0.06543 },
		{ 0.0, 0.00853, 0.2873 },
		{ 0.0, 0.00853, -0.21985 },
		{ -0.0, 1.69016, 0.21986 },
		{ 0.0, 1.75005, -0.02323 },
		{ 0.26395, 0.00853, 0.07559 },
		{ -0.29077, 0.00853, -0.03154 },
		{ 0.26686, 1.44191, 0.06391 },
		{ -0.27044, 1.42596, 0.06391 },
		{ 0.0, 0.00853, 0.0 },
		{ 0.0, 1.7837, 0.10032 },
		{ 0.18368, 1.12971, 0.23308 },
		{ 0.18368, 1.12971, -0.16557 },
		{ -0.23208, 1.12971, 0.21986 },
		{ -0.18368, 1.12971, -0.16557 },
		{ -0.05467, 1.22532, 0.4318 },
		{ 0.0, 1.12971, -0.21985 },
		{ 0.31219, 1.12971, 0.0325 },
		{ -0.30222, 1.12971, 0.0 }
	},
	placement = {
		translation = { 0.0, 0.15, 0.0 },
		orientation = { 1.0, 0.0, 0.0, 0.0 }
	},
	mass = 70.0,
}

For completeness, here's the source file for the application (pardon the Windows-ness please, there is no excuse):

Code: Select all

#include <libxml/xmlreader.h>
#include <libxml/parser.h>
#include <libxml/tree.h>

#include <lua.h>
#include <lauxlib.h>
#include <lualib.h>

#define TEST_COUNT 100000

struct TestDoc
{
	char *pContents;
	long nSize;

	TestDoc(char *p, long n) : pContents(p), nSize(n) {};
	~TestDoc() { delete pContents; };
};


TestDoc *ReadFileContents(const char *pFilename, bool bBinary)
{
	FILE *f = fopen(pFilename, bBinary ? "rb" : "r");
	char *pContents = NULL;
	long nSize = 0;
	if (f)
	{
		fseek(f, 0, SEEK_END);
		nSize = ftell(f);
		fseek(f, 0, SEEK_SET);
		pContents = new char[nSize+1];
		memset(pContents, 0, nSize+1);
		fread(pContents, 1, nSize, f);
		fclose(f);
	}

	return new TestDoc(pContents, nSize);
}

void ProcessSAXNode(xmlTextReaderPtr reader)
{
	const xmlChar *name = xmlTextReaderConstName(reader);
	const xmlChar *value = xmlTextReaderConstValue(reader);
}

void XmlSaxParseDoc(TestDoc &pDoc)
{
	xmlTextReaderPtr reader = xmlReaderForDoc(BAD_CAST pDoc.pContents, NULL, NULL, XML_PARSE_NOENT | XML_PARSE_NOCDATA | XML_PARSE_NOXINCNODE);
	if (reader != NULL)
	{
		int ret = xmlTextReaderRead(reader);
		while (ret == 1) 
		{
			ProcessSAXNode(reader);
			ret = xmlTextReaderRead(reader);
		}
		xmlFreeTextReader(reader);
		if (ret != 0)
		{
			fprintf(stderr, "Failed to parse\n");
		}
	}
	else
		fprintf(stderr, "Unable to create reader\n");
}

long XmlSaxTest(TestDoc &pDoc)
{
	long nStart = GetTickCount();

	for (int i = 0; i < TEST_COUNT; i++)
		XmlSaxParseDoc(pDoc);

	long nEnd = GetTickCount();

	return nEnd - nStart;
}

long XmlDomTest(TestDoc &pDoc)
{
	long nStart = GetTickCount();

	for (int i = 0; i < TEST_COUNT; i++)
	{
		xmlDocPtr doc = xmlReadDoc(BAD_CAST pDoc.pContents, NULL, NULL, XML_PARSE_NOENT | XML_PARSE_NOCDATA | XML_PARSE_NOXINCNODE);
		if (doc == NULL)
		{
			fprintf(stderr, "Failed to parse\n");
			break;
		}
		xmlFreeDoc(doc);
	}

	long nEnd = GetTickCount();

	return nEnd - nStart;
}

long LuaTest(lua_State *L, TestDoc &pDoc)
{
	long nStart = GetTickCount();

	for (int i = 0; i < TEST_COUNT; i++)
	{
		if (!luaL_loadbuffer(L, pDoc.pContents, pDoc.nSize, NULL))
		{
			lua_newtable(L);
			lua_setfenv(L, -2);

			if (lua_pcall(L, 0, 1, 0))
			{
				const char *s = lua_tostring(L, -1);
				lua_pop(L, 1);
				fprintf(stderr, s);
				break;
			}

			lua_pop(L, 1);
		}
		else
		{
			const char *s = lua_tostring(L, -1);
			lua_pop(L, 1);
			fprintf(stderr, s);
			break;
		}
	}

	long nEnd = GetTickCount();

	return nEnd - nStart;
}


//int APIENTRY _tWinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPTSTR lpCmdLine, int nCmdShow)
int main(int argc, char **argv)
{
	// XML //////////////////////////////////////

	// Get XML file contents
	TestDoc *pXmlDoc = ReadFileContents("test.xml", false);

	xmlInitParser();

	// XML SAX
	long nXmlSaxResult = XmlSaxTest(*pXmlDoc);

	// XML DOM
	long nXmlDomResult = XmlDomTest(*pXmlDoc);

	xmlCleanupParser();
	delete pXmlDoc;

	// LUA //////////////////////////////////////

	// Get Lua file contents
	TestDoc *pLuaDoc = ReadFileContents("test.lua", true);
	TestDoc *pLuacDoc = ReadFileContents("test.luac", true);

	lua_State *L = lua_open();

	// Uncompiled
	long nLuaResult = LuaTest(L, *pLuaDoc);

	// Compiled
	long nLuacResult = LuaTest(L, *pLuacDoc);

	lua_close(L);
	delete pLuaDoc;
	delete pLuacDoc;

	printf("Xml SAX: %u\n", nXmlSaxResult);
	printf("Xml DOM: %u\n", nXmlDomResult);
	printf("Lua uncompiled: %u\n", nLuaResult);
	printf("Lua compiled: %u\n", nLuacResult);

	return 0;
}

I ran the test in four modes (each ran 100000 times in a loop):

• Using the XML parser in SAX mode, which simply runs through the XML tree and issues callbacks for each node. Faster but more painful to program with in most cases. For each node, the callback did nothing but retrieve the node name and value from internal structures.
• Using the XML parser in DOM mode, which constructs a parsed tree from the XML file.
• Using the Lua parser. Processing the lua file consists of loading/parsing the file contents, evaluating them and returning the created table. The table was simply discarded.
• Using the Lua parser, but with a .luac file, i.e. one that was identical to the previous run but had been compiled ahead of time using the Lua compiler.

Finally, the results:
XML SAX: 17485 ms
XML DOM: 18843 ms
Lua uncompiled: 14282 ms
Lua compiled: 2703 ms

The fact that the Lua file is way less verbose but still quite readable is also a big plus in my book. I would like to note that the file was not that big. I did not have a larger XML/Lua file pair at hand and according to my experience with bigger Lua files, I do not think that would change the general direction of the results too much.

I am now off to renovate my exporter...

[xavier]

If load times are an issue you should probably consider writing and reading memory images of your data. XML is not intended (no matter what the COLLADA people, for instance, might try to tell you) as a runtime data format for games. As for Lua, it's a Turing-complete scripting language. Both are ideal during development, but like I said, if performance is a key concern, you shouldn't be parsing script or XML at load time at all.

[syedhs]

If you are concern about performance, why not convert the vertices/indices information into one binary file? Probably, a very quick one consist of the size of the information, and information itself. The processing cost will consist only of allocating the proper data structure, one file reading, and one memcpy - done.

[tp]

Obviously, neither method is the fastest possible. There are other reasons for keeping resources readable, and many other resource types besides a collision mesh (that is a good candidate for more efficient serialization). Besides, an improvement in loading times of about an order of magnitude is probably an acceptable solution to the average user.

As for Lua, it's a Turing-complete scripting language. Both are ideal during development, but like I said, if performance is a key concern, you shouldn't be parsing script or XML at load time at all.

As far as Lua is concerned, it originated as a language for this purpose, and the developers have, as far as I am able to tell, been making sure that it is a competitive technology for that purpose still. This whole topic is about the performance of Lua and XML, for people interested in using both or either. It is obvious there are alternatives out there.

[reptor]

If load times are an issue you should probably consider writing and reading memory images of your data. XML is not intended (no matter what the COLLADA people, for instance, might try to tell you) as a runtime data format for games. As for Lua, it's a Turing-complete scripting language. Both are ideal during development, but like I said, if performance is a key concern, you shouldn't be parsing script or XML at load time at all.

+1

[Kojack]

Yay for Lua.

I wonder if LuaJIT would improve it much over the compiled version. I'm not sure if it helps for a purely data loading script (probably not much).

[Jabberwocky]

If load times are an issue you should probably consider writing and reading memory images of your data.

Hey Xavier,

I'm curious about your experiences using memory images. Have you used this technique with complex classes made up of STL containers? Or have you had to write your own non-STL data structures to get this to work?

The reason I ask is because with STL, you're not going to have a contiguous block of memory you can easily just binary dump to file (at least in it's normal usage - maybe there's some tricks with custom memory allocators?) Otherwise, you'd need some relatively elaborate code, somewhat akin to an XML parser, which instead reads/writes binary instead of text. Whereas switching to custom non-STL data structures would suck for a lot of reasons.

Right now I use XML for both my initialization data and my saved game files. I can easily add some compression to keep file sizes down. Using XML just has the side effect of longer load times, as discussed above, which isn't a critical problem, but faster load times would be nice.

[xavier]

Issues like that with STL are why I moved to memory images wherever possible. When a class uses a container type of data structure, that data gets serialized out (when written) in a contiguous block and on load, the pointer fixup puts everything back how it needs to be, and now those container elements are contiguous again.

[sizeak]

Issues like that with STL are why I moved to memory images wherever possible. When a class uses a container type of data structure, that data gets serialized out (when written) in a contiguous block and on load, the pointer fixup puts everything back how it needs to be, and now those container elements are contiguous again.

+1. This is what I do both with ogre and my own renderer