fix(filesystem): Reject BIG archives cut off before the directory

Core/GameEngine/CMakeLists.txt

@@ -15,6 +15,7 @@ set(GAMEENGINE_SRC
 #    Include/Common/BattleHonors.h
 #    Include/Common/BezFwdIterator.h
 #    Include/Common/BezierSegment.h
+    Include/Common/BIGFileSystem.h
 #    Include/Common/BitFlags.h
 #    Include/Common/BitFlagsIO.h
 #    Include/Common/BorderColors.h
@@ -644,6 +645,7 @@ set(GAMEENGINE_SRC
     Source/Common/System/ArchiveFile.cpp
     Source/Common/System/ArchiveFileSystem.cpp
     Source/Common/System/AsciiString.cpp
+    Source/Common/System/BIGFileSystem.cpp
 #    Source/Common/System/BuildAssistant.cpp
 #    Source/Common/System/CriticalSection.cpp
 #    Source/Common/System/DataChunk.cpp

Core/GameEngine/Include/Common/ArchiveFile.h

@@ -47,26 +47,32 @@ class File;
 class ArchiveFile
 {
 public:
-	ArchiveFile();
-	virtual ~ArchiveFile();
+	ArchiveFile(AsciiString name, AsciiString path);
+	~ArchiveFile();
 
-	virtual Bool					getFileInfo( const AsciiString& filename, FileInfo *fileInfo) const = 0;	///< fill in the fileInfo struct with info about the file requested.
-	virtual File*					openFile( const Char *filename, Int access = 0) = 0;	///< Open the specified file within the archive file
-	virtual void					closeAllFiles() = 0;									///< Close all file opened in this archive file
-	virtual AsciiString		getName() = 0;												///< Returns the name of the archive file
-	virtual AsciiString		getPath() = 0;												///< Returns full path and name of archive file
-	virtual void					setSearchPriority( Int new_priority ) = 0;	///< Set this archive file's search priority
-	virtual void					close() = 0;													///< Close this archive file
+	Bool					getFileInfo( const AsciiString& filename, FileInfo *fileInfo) const;	///< fill in the fileInfo struct with info about the file requested.
+	File*					openFile( const Char *filename, Int access = 0);	///< Open the specified file within the archive file
+	void					closeAllFiles();									///< Close all file opened in this archive file
+	AsciiString		getName();												///< Returns the name of the archive file
+	AsciiString		getPath();												///< Returns full path and name of archive file
+	void					setSearchPriority( Int new_priority );	///< Set this archive file's search priority
+	void					close();													///< Close this archive file
 	void									attachFile(File *file);
 
 	void									getFileListInDirectory(const AsciiString& currentDirectory, const AsciiString& originalDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const;
-	void									getFileListInDirectory(const DetailedArchivedDirectoryInfo *dirInfo, const AsciiString& currentDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const;
 
 	void									addFile(const AsciiString& path, const ArchivedFileInfo *fileInfo); ///< add this file to our directory tree.
 
 protected:
+	void									getFileListInDirectory(const DetailedArchivedDirectoryInfo *dirInfo, const AsciiString& currentDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const;
 	const ArchivedFileInfo *		getArchivedFileInfo(const AsciiString& filename) const;	///< return the ArchivedFileInfo from the directory tree.
 
+protected:
 	File *m_file; ///< file pointer to the archive file on disk.  Kept open so we don't have to continuously open and close the file all the time.
+	FileInfo m_fileInfo; ///< file info of the archive file on disk.
+	AsciiString m_name; ///< archive file name
+	AsciiString m_path; ///< archive file path
+
+private:
 	DetailedArchivedDirectoryInfo m_rootDirectory;
 };

Core/GameEngine/Include/Common/ArchiveFileSystem.h

@@ -141,12 +141,9 @@ class ArchiveFileSystem : public SubsystemInterface
 	void					getFileListInDirectory(const AsciiString& currentDirectory, const AsciiString& originalDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const; ///< search the given directory for files matching the searchName (egs. *.ini, *.rep).  Possibly search subdirectories.  Scans each Archive file.
 	Bool					getFileInfo(const AsciiString& filename, FileInfo *fileInfo, FileInstance instance = 0) const; ///< see FileSystem.h
 
-	virtual Bool	loadBigFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite = FALSE) = 0;
+	virtual Bool	loadArchiveFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite = FALSE) = 0;
 
-	// Unprotected this for copy-protection routines
-	ArchiveFile* getArchiveFile(const AsciiString& filename, FileInstance instance = 0) const;
-
-	void loadMods();
+	virtual void loadMods() = 0;
 
 	ArchivedDirectoryInfo* friend_getArchivedDirectoryInfo(const Char* directory);
 
@@ -160,6 +157,7 @@ class ArchiveFileSystem : public SubsystemInterface
 		AsciiString lastToken; ///< Synonymous for file name if the search directory was a file path
 	};
 
+	ArchiveFile* getArchiveFile(const AsciiString& filename, FileInstance instance = 0) const;
 	ArchivedDirectoryInfoResult getArchivedDirectoryInfo(const Char* directory);
 
 	virtual void loadIntoDirectoryTree(ArchiveFile *archiveFile, Bool overwrite = FALSE, Bool sortedByName = FALSE);	///< load the archive file's header information and apply it to the global archive directory tree. sortedByName inserts by archive filename order (addon number convention), ignoring overwrite.

Core/GameEngineDevice/Include/Win32Device/Common/Win32BIGFileSystem.h → Core/GameEngine/Include/Common/BIGFileSystem.h

@@ -22,19 +22,19 @@
 //																																						//
 ////////////////////////////////////////////////////////////////////////////////
 
-//////// Win32BIGFileSystem.h ///////////////////////////
+//////// BIGFileSystem.h ///////////////////////////
 // Bryan Cleveland, August 2002
 /////////////////////////////////////////////////////////////
 
 #pragma once
 
 #include "Common/ArchiveFileSystem.h"
 
-class Win32BIGFileSystem : public ArchiveFileSystem
+class BIGFileSystem : public ArchiveFileSystem
 {
 public:
-	Win32BIGFileSystem();
-	virtual ~Win32BIGFileSystem() override;
+	BIGFileSystem();
+	virtual ~BIGFileSystem() override;
 
 	virtual void init() override;
 	virtual void update() override;
@@ -49,7 +49,9 @@ class Win32BIGFileSystem : public ArchiveFileSystem
 	virtual void closeArchiveFile(const Char *filename) override;
 	virtual void closeAllFiles() override;															///< Close all files associated with ArchiveFiles
 
-	virtual Bool loadBigFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite = FALSE) override;
+	virtual Bool loadArchiveFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite = FALSE) override;
+
+	virtual void loadMods() override;
 protected:
 
 };

Core/GameEngine/Include/Common/FileSystem.h

@@ -109,6 +109,14 @@ typedef UnsignedByte FileInstance;
 
 struct FileInfo {
 
+	FileInfo()
+		: sizeHigh(0)
+		, sizeLow(0)
+		, timestampHigh(0)
+		, timestampLow(0)
+	{
+	}
+
 	Int64 size() const { return (Int64)sizeHigh << 32 | sizeLow; }
 	Int64 timestamp() const { return (Int64)timestampHigh << 32 | timestampLow; }
 

Core/GameEngine/Source/Common/System/ArchiveFile.cpp

@@ -30,8 +30,12 @@
 
 #include "Common/ArchiveFile.h"
 #include "Common/ArchiveFileSystem.h"
+#include "Common/LocalFileSystem.h"
 #include "Common/file.h"
+#include "Common/LocalFile.h"
 #include "Common/PerfTimer.h"
+#include "Common/RAMFile.h"
+#include "Common/StreamingArchiveFile.h"
 
 
 // checks to see if str matches searchString.  Search string is done in the
@@ -89,8 +93,10 @@ ArchiveFile::~ArchiveFile()
 	}
 }
 
-ArchiveFile::ArchiveFile()
+ArchiveFile::ArchiveFile(AsciiString name, AsciiString path)
 	: m_file(nullptr)
+	, m_name(name)
+	, m_path(path)
 {
 }
 
@@ -106,13 +112,11 @@ void ArchiveFile::addFile(const AsciiString& path, const ArchivedFileInfo *fileI
 	while (!token.isEmpty())
 	{
 		DetailedArchivedDirectoryInfoMap::iterator tempiter = dirInfo->m_directories.find(token);
-		if (tempiter == dirInfo->m_directories.end())
-		{
+		if (tempiter == dirInfo->m_directories.end()) {
 			dirInfo = &(dirInfo->m_directories[token]);
 			dirInfo->m_directoryName = token;
 		}
-		else
-		{
+		else {
 			dirInfo = &tempiter->second;
 		}
 
@@ -132,18 +136,12 @@ void ArchiveFile::getFileListInDirectory(const AsciiString& currentDirectory, co
 	tokenizer.nextToken(&token, "\\/");
 
 	while (!token.isEmpty()) {
-
 		DetailedArchivedDirectoryInfoMap::const_iterator it = dirInfo->m_directories.find(token);
-		if (it != dirInfo->m_directories.end())
-		{
-			dirInfo = &it->second;
-		}
-		else
-		{
-			// if the directory doesn't exist, then there aren't any files to be had.
+		// if the directory doesn't exist, then there aren't any files to be had.
+		if (it == dirInfo->m_directories.end())
 			return;
-		}
 
+		dirInfo = &it->second;
 		tokenizer.nextToken(&token, "\\/");
 	}
 
@@ -153,33 +151,31 @@ void ArchiveFile::getFileListInDirectory(const AsciiString& currentDirectory, co
 void ArchiveFile::getFileListInDirectory(const DetailedArchivedDirectoryInfo *dirInfo, const AsciiString& currentDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const
 {
 	DetailedArchivedDirectoryInfoMap::const_iterator diriter = dirInfo->m_directories.begin();
-	while (diriter != dirInfo->m_directories.end()) {
+	for (; diriter != dirInfo->m_directories.end(); ++diriter) {
 		const DetailedArchivedDirectoryInfo *tempDirInfo = &(diriter->second);
-		AsciiString tempdirname;
-		tempdirname = currentDirectory;
-		if ((!tempdirname.isEmpty()) && (!tempdirname.endsWith("\\"))) {
-			tempdirname.concat('\\');
+		AsciiString tempDirName = currentDirectory;
+		if (!tempDirName.isEmpty() && !tempDirName.endsWith("\\")) {
+			tempDirName.concat('\\');
 		}
-		tempdirname.concat(tempDirInfo->m_directoryName);
-		getFileListInDirectory(tempDirInfo, tempdirname, searchName, filenameList, searchSubdirectories);
-		diriter++;
+		tempDirName.concat(tempDirInfo->m_directoryName);
+		getFileListInDirectory(tempDirInfo, tempDirName, searchName, filenameList, searchSubdirectories);
 	}
 
 	ArchivedFileInfoMap::const_iterator fileiter = dirInfo->m_files.begin();
-	while (fileiter != dirInfo->m_files.end()) {
-		if (SearchStringMatches(fileiter->second.m_filename, searchName)) {
+	for (; fileiter != dirInfo->m_files.end(); ++fileiter) {
+		const ArchivedFileInfo &fileInfo = fileiter->second;
+		if (SearchStringMatches(fileInfo.m_filename, searchName)) {
 			AsciiString tempfilename;
 			tempfilename = currentDirectory;
-			if ((!tempfilename.isEmpty()) && (!tempfilename.endsWith("\\"))) {
+			if (!tempfilename.isEmpty() && !tempfilename.endsWith("\\")) {
 				tempfilename.concat('\\');
 			}
-			tempfilename.concat(fileiter->second.m_filename);
+			tempfilename.concat(fileInfo.m_filename);
 			if (filenameList.find(tempfilename) == filenameList.end()) {
 				// only insert into the list if its not already in there.
 				filenameList.insert(tempfilename);
 			}
 		}
-		fileiter++;
 	}
 }
 
@@ -189,7 +185,18 @@ void ArchiveFile::attachFile(File *file)
 		m_file->close();
 		m_file = nullptr;
 	}
+
 	m_file = file;
+
+	if (m_file != nullptr) {
+		if (!TheLocalFileSystem->getFileInfo(m_file->getName(), &m_fileInfo)) {
+			DEBUG_CRASH(("ArchiveFile::attachFile - Cannot query file info for '%s' unexpectedly", m_file->getName()));
+			m_fileInfo = FileInfo();
+		}
+	}
+	else {
+		m_fileInfo = FileInfo();
+	}
 }
 
 const ArchivedFileInfo * ArchiveFile::getArchivedFileInfo(const AsciiString& filename) const
@@ -201,7 +208,7 @@ const ArchivedFileInfo * ArchiveFile::getArchivedFileInfo(const AsciiString& fil
 	tokenizer.toLower();
 	tokenizer.nextToken(&token, "\\/");
 
-	while (!token.find('.') || tokenizer.find('.'))
+	while (!tokenizer.isEmpty())
 	{
 		DetailedArchivedDirectoryInfoMap::const_iterator it = dirInfo->m_directories.find(token);
 		if (it != dirInfo->m_directories.end())
@@ -227,3 +234,116 @@ const ArchivedFileInfo * ArchiveFile::getArchivedFileInfo(const AsciiString& fil
 	}
 
 }
+
+//============================================================================
+// ArchiveFile::openFile
+//============================================================================
+
+File* ArchiveFile::openFile( const Char *filename, Int access )
+{
+	const ArchivedFileInfo *fileInfo = getArchivedFileInfo(AsciiString(filename));
+
+	if (fileInfo == nullptr) {
+		return nullptr;
+	}
+
+	RAMFile *ramFile = nullptr;
+
+	if (BitIsSet(access, File::STREAMING))
+		ramFile = newInstance( StreamingArchiveFile );
+	else
+		ramFile = newInstance( RAMFile );
+
+	ramFile->deleteOnClose();
+	if (ramFile->openFromArchive(m_file, fileInfo->m_filename, fileInfo->m_offset, fileInfo->m_size) == FALSE) {
+		ramFile->close();
+		ramFile = nullptr;
+		return nullptr;
+	}
+
+	if ((access & File::WRITE) == 0) {
+		// requesting read only access. Just return the RAM file.
+		return ramFile;
+	}
+
+	// whoever is opening this file wants write access, so copy the file to the local disk
+	// and return that file pointer.
+
+	constexpr size_t bufferSize = 0;
+	File *localFile = TheLocalFileSystem->openFile(filename, access, bufferSize);
+	if (localFile != nullptr) {
+		ramFile->copyDataToFile(localFile);
+	}
+
+	ramFile->close();
+	ramFile = nullptr;
+
+	return localFile;
+}
+
+//============================================================================
+// ArchiveFile::closeAllFiles
+//============================================================================
+
+void ArchiveFile::closeAllFiles()
+{
+
+}
+
+//============================================================================
+// ArchiveFile::getName
+//============================================================================
+
+AsciiString ArchiveFile::getName()
+{
+	return m_name;
+}
+
+//============================================================================
+// ArchiveFile::getPath
+//============================================================================
+
+AsciiString ArchiveFile::getPath()
+{
+	return m_path;
+}
+
+//============================================================================
+// ArchiveFile::setSearchPriority
+//============================================================================
+
+void ArchiveFile::setSearchPriority( Int new_priority )
+{
+
+}
+
+//============================================================================
+// ArchiveFile::close
+//============================================================================
+
+void ArchiveFile::close()
+{
+
+}
+
+//============================================================================
+// ArchiveFile::getFileInfo
+//============================================================================
+
+Bool ArchiveFile::getFileInfo(const AsciiString& filename, FileInfo *fileInfo) const
+{
+	const ArchivedFileInfo *archivedFileInfo = getArchivedFileInfo(filename);
+
+	if (archivedFileInfo == nullptr) {
+		return FALSE;
+	}
+
+	// fill in the size info.  Since the size can't be bigger than a JUNK file, the high int will always be 0.
+	fileInfo->sizeHigh = 0;
+	fileInfo->sizeLow = archivedFileInfo->m_size;
+	// the timestamp is taken from the archive file.
+	fileInfo->timestampHigh = m_fileInfo.timestampHigh;
+	fileInfo->timestampLow = m_fileInfo.timestampLow;
+
+	return TRUE;
+}