refactor tokenizing

tools/quake3/common/scriplib.cpp

@@ -27,6 +27,7 @@
 #include "qpathops.h"
 #include "scriplib.h"
 #include "vfs.h"
+#include <list>
 
 /*
    =============================================================================
@@ -38,19 +39,32 @@
 
 struct script_t
 {
-	char filename[1024];
+	CopiedString filename;
 	char    *buffer;
-	const char *script_p, *end_p;
+	const char *it, *end;
 	int line;
+	/* buffer is optional: != nullptr == own it  */
+	script_t( const char *filename, char *buffer, char *start, int size ) :
+		filename( filename ),
+		buffer( buffer ),
+		it( start ),
+		end( start + size ),
+		line( 1 )
+	{}
+	script_t() = delete;
+	script_t( const script_t& ) = delete;
+	script_t( script_t&& ) noexcept = delete;
+	script_t& operator=( const script_t& ) = delete;
+	script_t& operator=( script_t&& ) noexcept = delete;
+	~script_t(){
+		free( buffer );
+	}
 };
 
-#define MAX_INCLUDES    8
-script_t scriptstack[MAX_INCLUDES];
-script_t    *script;
-int scriptline;
+std::list<script_t> scriptstack;
 
+int scriptline;
 char token[MAXTOKEN];
-bool endofscript;
 bool tokenready;                     // only true if UnGetToken was just called
 
 /*
@@ -58,35 +72,23 @@ bool tokenready;                     // only true if UnGetToken was just called
    AddScriptToStack
    ==============
  */
-void AddScriptToStack( const char *filename, int index ){
-	int size;
+static void AddScriptToStack( const char *filename, int index, bool verbose ){
 	void* buffer;
-
-	script++;
-	if ( script == &scriptstack[MAX_INCLUDES] ) {
-		Error( "script file exceeded MAX_INCLUDES" );
-	}
-	strcpy( script->filename, ExpandPath( filename ) );
-
-	size = vfsLoadFile( script->filename, &buffer, index );
+	const int size = vfsLoadFile( filename, &buffer, index );
 
 	if ( size == -1 ) {
-		Sys_FPrintf( SYS_WRN, "Script file %s was not found\n", script->filename );
-		script--;
+		Sys_FPrintf( SYS_WRN, "Script file %s was not found\n", filename );
 	}
 	else
 	{
-		if ( index > 0 ) {
-			Sys_Printf( "entering %s (%d)\n", script->filename, index + 1 );
-		}
-		else{
-			Sys_Printf( "entering %s\n", script->filename );
+		if( verbose ){
+			if ( index > 0 )
+				Sys_Printf( "entering %s (%d)\n", filename, index + 1 );
+			else
+				Sys_Printf( "entering %s\n", filename );
 		}
 
-		script->buffer = void_ptr( buffer );
-		script->line = 1;
-		script->script_p = script->buffer;
-		script->end_p = script->buffer + size;
+		scriptstack.emplace_back( filename, void_ptr( buffer ), void_ptr( buffer ), size );
 	}
 }
 
@@ -96,40 +98,9 @@ void AddScriptToStack( const char *filename, int index ){
    LoadScriptFile
    ==============
  */
-void LoadScriptFile( const char *filename, int index ){
-	script = scriptstack;
-	AddScriptToStack( filename, index );
-
-	endofscript = false;
-	tokenready = false;
-}
-/* &unload current; for autopacker */
-void SilentLoadScriptFile( const char *filename, int index ){
-	int size;
-
-	if ( script->buffer != NULL && !endofscript ) {
-		free( script->buffer );
-		script->buffer = NULL;
-	}
-
-	script = scriptstack;
-
-	script++;
-	if ( script == &scriptstack[MAX_INCLUDES] ) {
-		Error( "script file exceeded MAX_INCLUDES" );
-	}
-	strcpy( script->filename, ExpandPath( filename ) );
-
-	size = vfsLoadFile( script->filename, (void **)&script->buffer, index );
-
-	if ( size == -1 ) {
-		Sys_FPrintf( SYS_WRN, "Script file %s was not found\n", script->filename );
-	}
-	script->line = 1;
-	script->script_p = script->buffer;
-	script->end_p = script->buffer + size;
-
-	endofscript = false;
+void LoadScriptFile( const char *filename, int index, bool verbose /* = true */ ){
+	scriptstack.clear();
+	AddScriptToStack( filename, index, verbose );
 	tokenready = false;
 }
 
@@ -139,19 +110,8 @@ void SilentLoadScriptFile( const char *filename, int index ){
    ==============
  */
 void ParseFromMemory( char *buffer, int size ){
-	script = scriptstack;
-	script++;
-	if ( script == &scriptstack[MAX_INCLUDES] ) {
-		Error( "script file exceeded MAX_INCLUDES" );
-	}
-	strcpy( script->filename, "memory buffer" );
-
-	script->buffer = buffer;
-	script->line = 1;
-	script->script_p = script->buffer;
-	script->end_p = script->buffer + size;
-
-	endofscript = false;
+	scriptstack.clear();
+	scriptstack.emplace_back( "memory buffer", nullptr, buffer, size );
 	tokenready = false;
 }
 
@@ -171,35 +131,26 @@ void ParseFromMemory( char *buffer, int size ){
    ==============
  */
 void UnGetToken( void ){
+	ENSURE( !tokenready && "Can't UnGetToken() twice in a row!" );
 	tokenready = true;
 }
 
 
-bool EndOfScript( bool crossline ){
+static bool EndOfScript( bool crossline ){
 	if ( !crossline ) {
 		Error( "Line %i is incomplete\nFile location be: %s\n", scriptline, g_strLoadedFileLocation );
 	}
 
-	if ( strEqual( script->filename, "memory buffer" ) ) {
-		endofscript = true;
-		return false;
-	}
+	scriptstack.pop_back();
 
-	if ( script->buffer == NULL ) {
-		Sys_Warning( "Attempt to free already freed script buffer\n" );
+	if ( scriptstack.empty() ) {
+		return false;
 	}
 	else{
-		free( script->buffer );
+		scriptline = scriptstack.back().line;
+		Sys_Printf( "returning to %s\n", scriptstack.back().filename.c_str() );
+		return GetToken( crossline );
 	}
-	script->buffer = NULL;
-	if ( script == scriptstack + 1 ) {
-		endofscript = true;
-		return false;
-	}
-	script--;
-	scriptline = script->line;
-	Sys_Printf( "returning to %s\n", script->filename );
-	return GetToken( crossline );
 }
 
 /*
@@ -208,11 +159,8 @@ bool EndOfScript( bool crossline ){
    ==============
  */
 bool GetToken( bool crossline ){
-	char    *token_p;
-
-
 	/* ydnar: dummy testing */
-	if ( script == NULL || script->buffer == NULL ) {
+	if ( scriptstack.empty() ) {
 		return false;
 	}
 
@@ -221,7 +169,9 @@ bool GetToken( bool crossline ){
 		return true;
 	}
 
-	if ( ( script->script_p >= script->end_p ) || ( script->script_p == NULL ) ) {
+	script_t& script = scriptstack.back();
+
+	if ( script.it >= script.end ) {
 		return EndOfScript( crossline );
 	}
 
@@ -229,85 +179,85 @@ bool GetToken( bool crossline ){
 // skip space
 //
 skipspace:
-	while ( script->script_p < script->end_p && *script->script_p <= 32 )
+	while ( script.it < script.end && *script.it <= 32 )
 	{
-		if ( *script->script_p++ == '\n' ) {
+		if ( *script.it++ == '\n' ) {
 			if ( !crossline ) {
 				Error( "Line %i is incomplete\nFile location be: %s\n", scriptline, g_strLoadedFileLocation );
 			}
-			script->line++;
-			scriptline = script->line;
+			script.line++;
+			scriptline = script.line;
 		}
 	}
 
-	if ( script->script_p >= script->end_p ) {
+	if ( script.it >= script.end ) {
 		return EndOfScript( crossline );
 	}
 
 	// ; # // comments
-	if ( *script->script_p == ';' || *script->script_p == '#'
-	     || ( script->script_p[0] == '/' && script->script_p[1] == '/' ) ) {
+	if ( *script.it == ';' || *script.it == '#'
+	     || ( script.it[0] == '/' && script.it[1] == '/' ) ) {
 		if ( !crossline ) {
 			Error( "Line %i is incomplete\nFile location be: %s\n", scriptline, g_strLoadedFileLocation );
 		}
-		while ( *script->script_p++ != '\n' )
-			if ( script->script_p >= script->end_p ) {
+		while ( *script.it++ != '\n' )
+			if ( script.it >= script.end ) {
 				return EndOfScript( crossline );
 			}
-		script->line++;
-		scriptline = script->line;
+		script.line++;
+		scriptline = script.line;
 		goto skipspace;
 	}
 
 	// /* */ comments
-	if ( script->script_p[0] == '/' && script->script_p[1] == '*' ) {
+	if ( script.it[0] == '/' && script.it[1] == '*' ) {
 		if ( !crossline ) {
 			Error( "Line %i is incomplete\nFile location be: %s\n", scriptline, g_strLoadedFileLocation );
 		}
-		script->script_p += 2;
-		while ( script->script_p[0] != '*' || script->script_p[1] != '/' )
+		script.it += 2;
+		while ( script.it[0] != '*' || script.it[1] != '/' )
 		{
-			if ( *script->script_p == '\n' ) {
-				script->line++;
-				scriptline = script->line;
+			if ( *script.it == '\n' ) {
+				script.line++;
+				scriptline = script.line;
 			}
-			script->script_p++;
-			if ( script->script_p >= script->end_p ) {
+			script.it++;
+			if ( script.it >= script.end ) {
 				return EndOfScript( crossline );
 			}
 		}
-		script->script_p += 2;
+		script.it += 2;
 		goto skipspace;
 	}
 
 //
 // copy token
 //
-	token_p = token;
+	char *token_p = token;
 
-	if ( *script->script_p == '"' ) {
+	if ( *script.it == '"' ) {
 		// quoted token
-		script->script_p++;
-		while ( *script->script_p != '"' )
+		script.it++;
+		while ( *script.it != '"' )
 		{
-			*token_p++ = *script->script_p++;
-			if ( script->script_p == script->end_p ) {
+			*token_p++ = *script.it++;
+			if ( script.it == script.end ) {
 				break;
 			}
-			if ( token_p == &token[MAXTOKEN] ) {
+			if ( token_p == token + MAXTOKEN ) {
 				Error( "Token too large on line %i\nFile location be: %s\n", scriptline, g_strLoadedFileLocation );
 			}
 		}
-		script->script_p++;
+		script.it++;
 	}
 	else{   // regular token
-		while ( *script->script_p > 32 && *script->script_p != ';' )
+		while ( *script.it > 32 && *script.it != ';' )
 		{
-			*token_p++ = *script->script_p++;
-			if ( script->script_p == script->end_p ) {
+			*token_p++ = *script.it++;
+			if ( script.it == script.end ) {
 				break;
 			}
-			if ( token_p == &token[MAXTOKEN] ) {
+			if ( token_p == token + MAXTOKEN ) {
 				Error( "Token too large on line %i\nFile location be: %s\n", scriptline, g_strLoadedFileLocation );
 			}
 		}
@@ -317,7 +267,7 @@ skipspace:
 
 	if ( strEqual( token, "$include" ) ) {
 		GetToken( false );
-		AddScriptToStack( token, 0 );
+		AddScriptToStack( token, 0, true );
 		return GetToken( crossline );
 	}
 
@@ -333,10 +283,8 @@ skipspace:
    ==============
  */
 bool TokenAvailable( void ) {
-	int oldLine;
-
 	/* save */
-	oldLine = scriptline;
+	const int oldLine = scriptline;
 
 	/* test */
 	if ( !GetToken( true ) ) {
@@ -362,7 +310,7 @@ void MatchToken( const char *match ) {
 	GetToken( true );
 
 	if ( !strEqual( token, match ) ) {
-		Error( "MatchToken( \"%s\" ) failed at line %i in file %s", match, scriptline, script->filename );
+		Error( "MatchToken( \"%s\" ) failed at line %i in file %s", match, scriptline, g_strLoadedFileLocation );
 	}
 }
 

tools/quake3/common/scriplib.h

@@ -19,23 +19,17 @@
    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
-// scriplib.h
+#pragma once
 
-#ifndef __CMDLIB__
-#include "../common/cmdlib.h"
-#endif
+#include "cmdlib.h"
 
 #define MAXTOKEN    1024
 
 extern char token[MAXTOKEN];
-extern char    *scriptbuffer,*script_p,*scriptend_p;
-extern int grabbed;
 extern int scriptline;
-extern bool endofscript;
 
 
-void LoadScriptFile( const char *filename, int index );
-void SilentLoadScriptFile( const char *filename, int index );
+void LoadScriptFile( const char *filename, int index, bool verbose = true );
 void ParseFromMemory( char *buffer, int size );
 
 bool GetToken( bool crossline );

tools/quake3/q3data/q3data.cpp

@@ -441,8 +441,7 @@ void ParseScript( void ){
 	{
 		do
 		{   // look for a line starting with a $ command
-			GetToken( true );
-			if ( endofscript ) {
+			if ( !GetToken( true ) ) {
 				return;
 			}
 			if ( token[0] == '$' ) {

tools/quake3/q3map2/autopk3.cpp

@@ -378,9 +378,6 @@ int pk3BSPMain( Args& args ){
 	}
 
 	//Parse Shader Files
-	/* hack */
-	endofscript = true;
-
 	for ( CopiedString& file : pk3Shaderfiles ){
 		bool wantShaderFile = false;
 		const String64 *excludedByShader = nullptr;
@@ -389,9 +386,7 @@ int pk3BSPMain( Args& args ){
 
 		/* load the shader */
 		const auto scriptFile = stream( g_game->shaderPath, '/', file );
-		SilentLoadScriptFile( scriptFile, 0 );
-		if( dbg )
-			Sys_Printf( "\n\tentering %s\n", file.c_str() );
+		LoadScriptFile( scriptFile, 0, dbg );
 
 		/* tokenize it */
 		/* check if shader file has to be excluded */
@@ -442,7 +437,7 @@ int pk3BSPMain( Args& args ){
 		}
 
 		/* tokenize it again */
-		SilentLoadScriptFile( scriptFile, 0 );
+		LoadScriptFile( scriptFile, 0, false );
 		while ( 1 )
 		{
 			/* test for end of file */
@@ -849,15 +844,11 @@ int repackBSPMain( Args& args ){
 	Sys_Printf( "\t\nParsing shaders....\n\n" );
 	StringOutputStream shaderText( 4096 );
 	StringOutputStream allShaders( 1048576 );
-	/* hack */
-	endofscript = true;
 
 	for ( const CopiedString& file : pk3Shaderfiles ){
 		/* load the shader */
 		const auto scriptFile = stream( g_game->shaderPath, '/', file );
-		SilentLoadScriptFile( scriptFile, 0 );
-		if ( dbg )
-			Sys_Printf( "\n\tentering %s\n", file.c_str() );
+		LoadScriptFile( scriptFile, 0, dbg );
 
 		/* tokenize it */
 		while ( 1 )