From db1f76996c85aac22c3dcca8b0d69918395f76e7 Mon Sep 17 00:00:00 2001 From: Doyle Thai Date: Mon, 26 Jun 2017 21:46:28 +1000 Subject: [PATCH] Add quicker filesize query, and ReadEntireFile() --- dqn.h | 185 +++++++++++++++++++++++++++++++++++++++------- dqn_unit_test.cpp | 63 ++++++++++++---- 2 files changed, 210 insertions(+), 38 deletions(-) diff --git a/dqn.h b/dqn.h index 23bb371..83bb5a0 100644 --- a/dqn.h +++ b/dqn.h @@ -60,6 +60,10 @@ // #DqnIni Simple INI Config File API (Public Domain lib by Mattias Gustavsson) // #DqnSprintf Cross-platform Sprintf Implementation (Public Domain lib stb_sprintf) +// TODO +// - Win32 +// - Get rid of reliance on MAX_PATH + //////////////////////////////////////////////////////////////////////////////// // Preprocessor Checks //////////////////////////////////////////////////////////////////////////////// @@ -1033,7 +1037,7 @@ typedef struct DqnFile #endif } DqnFile; -// NOTE: W(ide) versions of functions only work on Win32, since Unix is UTF-8 compatible. +// NOTE: W(ide) versions of functions only work on Win32, since Unix is already UTF-8 compatible. // Open a handle for file read and writing. Deleting files does not need a handle. Handles should be // closed before deleting files otherwise the OS may not be able to delete the file. @@ -1048,9 +1052,25 @@ DQN_FILE_SCOPE size_t DqnFile_Write(const DqnFile *const file, u8 *const buffer, // return: The number of bytes read. 0 if invalid args or it failed to read. DQN_FILE_SCOPE size_t DqnFile_Read (const DqnFile *const file, u8 *const buffer, const size_t numBytesToRead); +// Read the entire file at path into the given buffer. To determine the necessary buffer size, use +// DqnFile_GetFileSize() +// buffer: The buffer to put data into. +// bufferSize: The size of the given buffer. +// bytesRead: (Optionally) Pass in a pointer to a size_t to get how many bytes of the buffer was +// used. Will be valid only if function returns true. This is basically the return value +// of DqnFile_Read() +// return: FALSE if insufficient bufferSize OR file access failure OR invalid args (i.e nullptr) +DQN_FILE_SCOPE bool DqnFile_ReadEntireFile(const char *const path, u8 *const buffer, + const size_t bufferSize, size_t *const bytesRead); + // File close invalidates the handle after it is called. DQN_FILE_SCOPE void DqnFile_Close(DqnFile *const file); +// size: Pass in a pointer to a size_t where the file size gets put into. It holds invalid data if +// function returns false. +DQN_FILE_SCOPE bool DqnFile_GetFileSize (const char *const path, size_t *const size); +DQN_FILE_SCOPE bool DqnFile_GetFileSizeW(const wchar_t *const path, size_t *const size); + // NOTE: You can't delete a file unless the handle has been closed to it on Win32. DQN_FILE_SCOPE bool DqnFile_Delete (const char *const path); DQN_FILE_SCOPE bool DqnFile_DeleteW(const wchar_t *const path); @@ -5792,9 +5812,34 @@ DQN_FILE_SCOPE char **DqnDirInternal_PlatformRead(const char *const dir, return list; } } + #endif // DQN_WIN32_PLATFORM #ifdef DQN_UNIX_PLATFORM +FILE_SCOPE bool DqnFileInternal_UnixGetFileSizeWithStat(const char *const path, size_t *const size) +{ + struct stat fileStat = {0}; + if (stat(path, &fileStat)) return false; + *size = fileStat.st_size; + + return true; +} + +FILE_SCOPE size_t DqnFileInternal_UnixGetFileSizeManual(FILE *const handle, const bool rewindHandle) +{ + u64 fileSizeInBytes = 0; + + DQN_ASSERT_HARD(handle); + char c = fgetc(handle); + while (c != EOF) + { + fileSizeInBytes++; + c = fgetc(handle); + } + if (rewindHandle) rewind(handle); + return fileSizeInBytes; +} + FILE_SCOPE bool DqnFileInternal_UnixOpen(const char *const path, DqnFile *const file, const u32 permissionFlags, @@ -5851,8 +5896,7 @@ FILE_SCOPE bool DqnFileInternal_UnixOpen(const char *const path, FILE *handle = fopen(path, mode); if (!handle) return false; - struct stat fileStat = {0}; - if (stat(path, &fileStat)) + if (!DqnFileInternal_UnixGetFileSizeWithStat(path, &file->size)) { // TODO(doyle): Logging fclose(handle); @@ -5863,16 +5907,10 @@ FILE_SCOPE bool DqnFileInternal_UnixOpen(const char *const path, file->permissionFlags = permissionFlags; // NOTE: Can occur in some instances where files are generated on demand, - // i.e. /proc/cpuinfo - if (fileStat.st_size == 0) - { - // TODO - file->size = fileStat.st_size; - } - else - { - file->size = fileStat.st_size; - } + // i.e. /proc/cpuinfo. But there can also be zero-byte files, we can't + // be sure. So manual check by counting bytes + if (file->size == 0) + file->size = DqnFileInternal_UnixGetFileSizeManual((FILE *)file->handle, true); return true; } @@ -5944,6 +5982,7 @@ DQN_FILE_SCOPE char **DqnDirInternal_PlatformRead(const char *const dir, return list; } } + #endif // DQN_UNIX_PLATFORM //////////////////////////////////////////////////////////////////////////////// @@ -5956,6 +5995,7 @@ bool DqnFile_Open(const char *const path, DqnFile *const file, if (!file || !path) return false; #if defined(DQN_WIN32_PLATFORM) + // TODO(doyle): MAX PATH is baad wchar_t widePath[MAX_PATH] = {0}; DqnWin32_UTF8ToWChar(path, widePath, DQN_ARRAY_COUNT(widePath)); return DqnFileInternal_Win32OpenW(widePath, file, permissionFlags, action); @@ -6061,6 +6101,35 @@ DQN_FILE_SCOPE size_t DqnFile_Read(const DqnFile *const file, u8 *const buffer, return numBytesRead; } +DQN_FILE_SCOPE bool DqnFile_ReadEntireFile(const char *const path, u8 *const buffer, + const size_t bufferSize, size_t *const bytesRead) +{ + if (!path || !buffer) return false; + + bool result = true; + size_t bytesReadInternal; + + // TODO(doyle): Logging + DqnFile file = {}; + result = DqnFile_Open(path, &file, DqnFilePermissionFlag_Read, DqnFileAction_OpenOnly); + if (!result) goto cleanup; + + if (file.size > bufferSize) + { + result = false; + goto cleanup; + } + + bytesReadInternal = DqnFile_Read(&file, buffer, file.size); + DQN_ASSERT_HARD(bytesReadInternal == file.size); + + if (bytesRead) *bytesRead = bytesReadInternal; + +cleanup: + DqnFile_Close(&file); + return result; +} + DQN_FILE_SCOPE void DqnFile_Close(DqnFile *const file) { if (file && file->handle) @@ -6076,11 +6145,74 @@ DQN_FILE_SCOPE void DqnFile_Close(DqnFile *const file) } } +#if defined(DQN_WIN32_PLATFORM) + DQN_COMPILE_ASSERT(sizeof(DWORD) == sizeof(u32)); + DQN_COMPILE_ASSERT(sizeof(size_t) >= sizeof(u64)); +#endif + +DQN_FILE_SCOPE bool DqnFile_GetFileSizeW(const wchar_t *const path, size_t *const size) +{ + +#if defined(DQN_WIN32_PLATFORM) + WIN32_FILE_ATTRIBUTE_DATA attribData = {}; + if (GetFileAttributesExW(path, GetFileExInfoStandard, &attribData)) + { + LARGE_INTEGER largeInt = {}; + largeInt.HighPart = attribData.nFileSizeHigh; + largeInt.LowPart = attribData.nFileSizeLow; + *size = largeInt.QuadPart; + return true; + } + +#elif defined(DQN_UNIX_PLATFORM) + // NOTE: Not supported on unix + DQN_ASSERT_HARD(DQN_INVALID_CODE_PATH); + +#endif + + return false; +} + +DQN_FILE_SCOPE bool DqnFile_GetFileSize(const char *const path, size_t *const size) +{ + if (!path || !size) return false; + + // TODO(doyle): Logging +#if defined(DQN_WIN32_PLATFORM) + // TODO(doyle): MAX PATH is baad + wchar_t widePath[MAX_PATH] = {0}; + DqnWin32_UTF8ToWChar(path, widePath, DQN_ARRAY_COUNT(widePath)); + return DqnFile_GetFileSizeW(widePath, size); + +#elif defined(DQN_UNIX_PLATFORM) + // TODO(doyle): Error logging + if (!DqnFileInternal_UnixGetFileSizeWithStat(path, size)) return false; + + // NOTE: 0 size can occur in some instances where files are generated on demand, + // i.e. /proc/cpuinfo + if (*size == 0) + { + // If stat fails, then do a manual byte count + DqnFile file = {}; + if (!DqnFileInternal_UnixOpen(path, &file, DqnFilePermissionFlag_Read, + DqnFileAction_OpenOnly)) + { + return false; + } + + *size = DqnFileInternal_UnixGetFileSizeManual((FILE *)file.handle, false); + DqnFile_Close(&file); + } + + return true; +#endif +} + DQN_FILE_SCOPE bool DqnFile_Delete(const char *const path) { if (!path) return false; - // TODO(doyle): Logging +// TODO(doyle): Logging #if defined(DQN_WIN32_PLATFORM) return DeleteFileA(path); @@ -6093,6 +6225,20 @@ DQN_FILE_SCOPE bool DqnFile_Delete(const char *const path) #endif } +DQN_FILE_SCOPE bool DqnFile_DeleteW(const wchar_t *const path) +{ + if (!path) return false; + + // TODO(doyle): Logging +#if defined(DQN_WIN32_PLATFORM) + return DeleteFileW(path); + +#elif defined(DQN_UNIX_PLATFORM) + DQN_ASSERT_HARD(DQN_INVALID_CODE_PATH); + return false; + +#endif +} //////////////////////////////////////////////////////////////////////////////// // XPlatform > #DqnDir Implementation //////////////////////////////////////////////////////////////////////////////// @@ -6546,17 +6692,6 @@ FILE_SCOPE void DqnPlatformInternal_UnixGetNumCoresAndThreads(u32 *const numCore DqnFile file = {}; DQN_ASSERT_HARD( DqnFile_Open("/proc/cpuinfo", &file, DqnFilePermissionFlag_Read, DqnFileAction_OpenOnly)); - - // NOTE: cpuinfo is generated by kernel when queried, so we don't actually know the size. - DQN_ASSERT_HARD(file.size == 0); - - char c = fgetc((FILE *)file.handle); - while(c != EOF) - { - file.size++; - c = fgetc((FILE *)file.handle); - } - rewind((FILE *)file.handle); DQN_ASSERT_HARD(file.size > 0); u8 *readBuffer = (u8 *)DqnMem_Calloc(file.size); diff --git a/dqn_unit_test.cpp b/dqn_unit_test.cpp index 2741def..2239176 100644 --- a/dqn_unit_test.cpp +++ b/dqn_unit_test.cpp @@ -1548,6 +1548,16 @@ void FileTest() expectedSize = fileStat.st_size; } + if (1) + { + // NOTE: cpuinfo is generated when queried, so a normal 'stat' + // should give us zero, but we fall back to manual byte checking + // which should give us the proper size. + size_t size = 0; + DQN_ASSERT(DqnFile_GetFileSize("/proc/cpuinfo", &size)); + DQN_ASSERT(size > 0); + } + #elif defined(DQN_WIN32_IMPLEMENTATION) { HANDLE handle = CreateFile(FILE_TO_OPEN, GENERIC_READ, 0, NULL, OPEN_EXISTING, @@ -1566,6 +1576,13 @@ void FileTest() } #endif + if (1) + { + size_t size = 0; + DQN_ASSERT_HARD(DqnFile_GetFileSize(FILE_TO_OPEN, &size)); + DQN_ASSERT_HARD(size == expectedSize); + } + DqnFile file = {}; DQN_ASSERT(DqnFile_Open(".clang-format", &file, (DqnFilePermissionFlag_Write | DqnFilePermissionFlag_Read), @@ -1641,23 +1658,43 @@ void FileTest() // Read data back in for (u32 i = 0; i < DQN_ARRAY_COUNT(fileNames); i++) { - u32 permissions = DqnFilePermissionFlag_Read; - DqnFile *file = files + i; - bool result = DqnFile_Open(fileNames[i], file, permissions, DqnFileAction_OpenOnly); - DQN_ASSERT(result); + // Manual read the file contents + { + u32 permissions = DqnFilePermissionFlag_Read; + DqnFile *file = files + i; + bool result = DqnFile_Open(fileNames[i], file, permissions, DqnFileAction_OpenOnly); + DQN_ASSERT(result); - u8 *buffer = (u8 *)DqnMemStack_Push(&memStack, file->size); - DQN_ASSERT(buffer); + u8 *buffer = (u8 *)DqnMemStack_Push(&memStack, file->size); + DQN_ASSERT(buffer); - size_t bytesRead = DqnFile_Read(&files[i], buffer, file->size); - DQN_ASSERT(bytesRead == file->size); + size_t bytesRead = DqnFile_Read(&files[i], buffer, file->size); + DQN_ASSERT(bytesRead == file->size); - // Verify the data is the same as we wrote out - DQN_ASSERT(DqnStr_Cmp((char *)buffer, (writeData[i])) == 0); + // Verify the data is the same as we wrote out + DQN_ASSERT(DqnStr_Cmp((char *)buffer, (writeData[i])) == 0); - // Delete when we're done with it - DQN_ASSERT(DqnMemStack_Pop(&memStack, buffer, file->size)); - DqnFile_Close(file); + // Delete when we're done with it + DQN_ASSERT(DqnMemStack_Pop(&memStack, buffer, file->size)); + DqnFile_Close(file); + } + + // Read using the ReadEntireFile api which doesn't need a file handle as an argument + { + size_t reqSize; + DQN_ASSERT(DqnFile_GetFileSize(fileNames[i], &reqSize)); + + u8 *buffer = (u8 *)DqnMemStack_Push(&memStack, reqSize); + DQN_ASSERT(buffer); + + size_t bytesRead = 0; + DQN_ASSERT(DqnFile_ReadEntireFile(fileNames[i], buffer, reqSize, &bytesRead)); + DQN_ASSERT(bytesRead == reqSize); + + // Verify the data is the same as we wrote out + DQN_ASSERT(DqnStr_Cmp((char *)buffer, (writeData[i])) == 0); + DQN_ASSERT(DqnMemStack_Pop(&memStack, buffer, reqSize)); + } DQN_ASSERT(DqnFile_Delete(fileNames[i])); }