/*****************************************************************************/

/* CascIndexFiles.cpp Copyright (c) Ladislav Zezula 2019 */

/*---------------------------------------------------------------------------*/

/* Index file support */

/*---------------------------------------------------------------------------*/

/* Date Ver Who Comment */

/* -------- ---- --- ------- */

/* 23.05.19 1.00 Lad Created */

/*****************************************************************************/

#define __CASCLIB_SELF__

#include "CascLib.h"

#include "CascCommon.h"

//-----------------------------------------------------------------------------

// Local variables

static LPCTSTR szAllowedHexChars = _T("0123456789aAbBcCdDeEfF");

static LPCTSTR szIndexFormat_V1 = _T("data.i%x%x");

static LPCTSTR szIndexFormat_V2 = _T("%02x%08x.idx");

// Limit for "orphaned" items - those that are in index files, but are not in ENCODING manifest

#define CASC_MAX_ORPHANED_ITEMS 0x100

typedef bool (*EKEY_ENTRY_CALLBACK)(TCascStorage * hs, CASC_INDEX_HEADER & InHeader, LPBYTE pbEKeyEntry);

//-----------------------------------------------------------------------------

// Local functions

// "data.iXY"

static bool IsIndexFileName_V1(LPCTSTR szFileName)

{

// Check if the name looks like a valid index file

return (_tcslen(szFileName) == 8 &&

_tcsnicmp(szFileName, _T("data.i"), 6) == 0 &&

_tcsspn(szFileName + 6, szAllowedHexChars) == 2);

}

static bool IsIndexFileName_V2(LPCTSTR szFileName)

{

// Check if the name looks like a valid index file

return (_tcslen(szFileName) == 14 &&

_tcsspn(szFileName, _T("0123456789aAbBcCdDeEfF")) == 0x0A &&

_tcsicmp(szFileName + 0x0A, _T(".idx")) == 0);

}

static bool IndexDirectory_OnFileFound(

LPCTSTR szFileName,

void * pvContext)

{

TCascStorage * hs = (TCascStorage *)pvContext;

PCASC_INDEX pIndexFile;

DWORD IndexVersion = 0;

DWORD IndexValue = 0;

// Auto-detect the format of the index file name

if(hs->szIndexFormat == NULL)

{

if(IsIndexFileName_V2(szFileName))

hs->szIndexFormat = szIndexFormat_V2;

else if(IsIndexFileName_V1(szFileName))

hs->szIndexFormat = szIndexFormat_V1;

else

return false;

}

if(hs->szIndexFormat == szIndexFormat_V2)

{

// Check the index file name format

if(!IsIndexFileName_V2(szFileName))

return false;

// Get the main index from the first two digits

if(ConvertStringToInt(szFileName + 0, 2, IndexValue) != ERROR_SUCCESS)

return false;

if(ConvertStringToInt(szFileName + 2, 8, IndexVersion) != ERROR_SUCCESS)

return false;

}

else if(hs->szIndexFormat == szIndexFormat_V1)

{

// Check the index file name format

if(!IsIndexFileName_V1(szFileName))

return false;

// Get the main index from the first two digits

if(ConvertStringToInt(szFileName + 6, 1, IndexValue) != ERROR_SUCCESS)

return false;

if(ConvertStringToInt(szFileName + 7, 1, IndexVersion) != ERROR_SUCCESS)

return false;

}

else

{

// Should never happen

assert(false);

return false;

}

// The index value must not be greater than 0x0F

if(IndexValue >= CASC_INDEX_COUNT)

return false;

pIndexFile = &hs->IndexFiles[IndexValue];

// If the new subindex is greater than the previous one, use this one instead

if(IndexVersion > pIndexFile->NewSubIndex)

{

pIndexFile->OldSubIndex = pIndexFile->NewSubIndex;

pIndexFile->NewSubIndex = IndexVersion;

}

else if(IndexVersion > pIndexFile->OldSubIndex)

{

pIndexFile->OldSubIndex = IndexVersion;

}

// Note: WoW6 only keeps last two index files

// Any additional index files are deleted at this point

return true;

}

static LPTSTR CreateIndexFileName(TCascStorage * hs, DWORD IndexValue, DWORD IndexVersion)

{

TCHAR szFullName[MAX_PATH];

TCHAR szPlainName[0x40];

// Sanity checks

assert(hs->szIndexFormat != NULL);

assert(hs->szIndexPath != NULL);

assert(IndexValue <= 0x0F);

// Create the full path

CascStrPrintf(szPlainName, _countof(szPlainName), hs->szIndexFormat, IndexValue, IndexVersion);

CombinePath(szFullName, _countof(szFullName), hs->szIndexPath, szPlainName, NULL);

// Return allocated path

return CascNewStr(szFullName);

}

static void SaveFileOffsetBitsAndEKeyLength(TCascStorage * hs, BYTE FileOffsetBits, BYTE EKeyLength)

{

if(hs->FileOffsetBits == 0)

hs->FileOffsetBits = FileOffsetBits;

assert(hs->FileOffsetBits == FileOffsetBits);

if(hs->EKeyLength == 0)

hs->EKeyLength = EKeyLength;

assert(hs->EKeyLength == EKeyLength);

}

// Verifies a guarded block - data availability and checksum match

static LPBYTE CaptureGuardedBlock1(LPBYTE pbFileData, LPBYTE pbFileEnd)

{

PFILE_INDEX_GUARDED_BLOCK pBlock = (PFILE_INDEX_GUARDED_BLOCK)pbFileData;

// Check the guarded block

if((pbFileData + sizeof(FILE_INDEX_GUARDED_BLOCK)) >= pbFileEnd)

return NULL;

if((pbFileData + sizeof(FILE_INDEX_GUARDED_BLOCK) + pBlock->BlockSize) > pbFileEnd)

return NULL;

// Verify the hash

if(hashlittle(pbFileData + sizeof(FILE_INDEX_GUARDED_BLOCK), pBlock->BlockSize, 0) != pBlock->BlockHash)

return NULL;

// Give the output

return (LPBYTE)(pBlock + 1);

}

// Second method of checking a guarded block; hash is calculated entry-by-entry.

// Unfortunately, due to implementation in hashlittle2(), we cannot calculate the hash of a continuous block

static LPBYTE CaptureGuardedBlock2(LPBYTE pbFileData, LPBYTE pbFileEnd, size_t EntryLength, PDWORD PtrBlockSize = NULL)

{

PFILE_INDEX_GUARDED_BLOCK pBlock = (PFILE_INDEX_GUARDED_BLOCK)pbFileData;

LPBYTE pbEntryPtr;

size_t EntryCount;

unsigned int HashBlizzGet = 0;

unsigned int HashHigh = 0;

unsigned int HashLow = 0;

// Check the guarded block. There must be enough bytes to contain FILE_INDEX_GUARDED_BLOCK

// and also the block length must not be NULL

if ((pbFileData + sizeof(FILE_INDEX_GUARDED_BLOCK)) >= pbFileEnd)

return NULL;

if (pBlock->BlockSize == 0 || (pbFileData + sizeof(FILE_INDEX_GUARDED_BLOCK) + pBlock->BlockSize) > pbFileEnd)

return NULL;

//

// Verify the hash from the Blizzard Downloader

//

pbEntryPtr = pbFileData + sizeof(FILE_INDEX_GUARDED_BLOCK);

EntryCount = pBlock->BlockSize / EntryLength;

for (size_t i = 0; i < EntryCount; i++)

{

hashlittle2(pbEntryPtr, EntryLength, &HashHigh, &HashLow);

pbEntryPtr += EntryLength;

}

// Verify hash

if (HashHigh == pBlock->BlockHash)

{

// Give the output

if (PtrBlockSize != NULL)

PtrBlockSize[0] = pBlock->BlockSize;

return (LPBYTE)(pBlock + 1);

}

//

// Verify the hash from the Blizzget tool, which calculates the hash differently

// https://github.com/d07RiV/blizzget/blob/master/src/ngdp.cpp

// Function void DataStorage::writeIndex()

//

pbEntryPtr = pbFileData + sizeof(FILE_INDEX_GUARDED_BLOCK);

EntryCount = pBlock->BlockSize / EntryLength;

for (size_t i = 0; i < EntryCount; i++)

{

HashBlizzGet = hashlittle(pbEntryPtr, EntryLength, HashBlizzGet);

pbEntryPtr += EntryLength;

}

// Verify hash

if (HashBlizzGet == pBlock->BlockHash)

{

// Give the output

if (PtrBlockSize != NULL)

PtrBlockSize[0] = pBlock->BlockSize;

return (LPBYTE)(pBlock + 1);

}

// Hash mismatch

return NULL;

}

// Third method of checking a guarded block; There is 32-bit hash, followed by EKey entry

// The hash covers the EKey entry plus one byte

static LPBYTE CaptureGuardedBlock3(LPBYTE pbFileData, LPBYTE pbFileEnd, size_t EntryLength)

{

PDWORD PtrEntryHash = (PDWORD)pbFileData;

DWORD EntryHash;

// Check the guarded block. There must be enough bytes to contain single entry and the hash

// Also, the hash must be present

if ((pbFileData + sizeof(DWORD) + EntryLength) >= pbFileEnd)

return NULL;

if (PtrEntryHash[0] == 0)

return NULL;

EntryHash = hashlittle(pbFileData + sizeof(DWORD), EntryLength+1, 0) | 0x80000000;

if(EntryHash != PtrEntryHash[0])

return NULL;

// Give the output

return (LPBYTE)(PtrEntryHash + 1);

}

/*

static bool CaptureIndexEntry(CASC_INDEX_HEADER & InHeader, PCASC_EKEY_ENTRY pEKeyEntry, LPBYTE pbEKeyEntry)

{

// Copy the EKey of the variable length

pbEKeyEntry = CaptureEncodedKey(pEKeyEntry->EKey, pbEKeyEntry, InHeader.EKeyLength);

// Copy the storage offset and encoded size

pEKeyEntry->StorageOffset = ConvertBytesToInteger_5(pbEKeyEntry);

pEKeyEntry->EncodedSize = ConvertBytesToInteger_4_LE(pbEKeyEntry + InHeader.StorageOffsetLength);

pEKeyEntry->Alignment = 0;

// We ignore items that have EncodedSize of 0x1E

return (pEKeyEntry->EncodedSize > FIELD_OFFSET(BLTE_ENCODED_HEADER, Signature));

}

static void InsertCKeyEntry(TCascStorage * hs, CASC_EKEY_ENTRY & EKeyEntry, DWORD Flags)

{

PCASC_CKEY_ENTRY pCKeyEntry;

// Multiple items with the same EKey in the index files may exist.

// Example: "2018 - New CASC\00001", EKey 37 89 16 5b 2d cc 71 c1 25 00 00 00 00 00 00 00

// Positions: 0x1D, 0x1E, 0x1F

// In that case, we only take the first one into account

// BREAK_ON_XKEY3(EKeyEntry.EKey, 0x09, 0xF3, 0xCD);

// If the item is not there yet, insert a new one

if((pCKeyEntry = FindCKeyEntry_EKey(hs, EKeyEntry.EKey)) == NULL)

{

// Insert a new entry to the array. DO NOT ALLOW enlarge array here

pCKeyEntry = (PCASC_CKEY_ENTRY)hs->CKeyArray.Insert(1, false);

if(pCKeyEntry == NULL)

return;

// Fill-in the information

ZeroMemory16(pCKeyEntry->CKey);

CopyMemory16(pCKeyEntry->EKey, EKeyEntry.EKey);

pCKeyEntry->StorageOffset = EKeyEntry.StorageOffset;

pCKeyEntry->TagBitMask = 0;

pCKeyEntry->ContentSize = CASC_INVALID_SIZE;

pCKeyEntry->EncodedSize = EKeyEntry.EncodedSize;

pCKeyEntry->Flags = CASC_CE_HAS_EKEY | CASC_CE_HAS_EKEY_PARTIAL;

pCKeyEntry->RefCount = 0;

pCKeyEntry->SpanCount = 1;

pCKeyEntry->Priority = 0;

// Insert the item to the EKey table

hs->EKeyMap.InsertObject(pCKeyEntry, pCKeyEntry->EKey);

}

else

{

// The entry already exists. True e.g. for ENCODING.

// Only copy the storage offset and sizes if not available yet

if(pCKeyEntry->StorageOffset == CASC_INVALID_OFFS64)

{

pCKeyEntry->StorageOffset = EKeyEntry.StorageOffset;

pCKeyEntry->EncodedSize = EKeyEntry.EncodedSize;

}

}

// Add the extra flag

pCKeyEntry->Flags |= Flags;

}

*/

static DWORD LoadIndexItems(TCascStorage * hs, CASC_INDEX_HEADER & InHeader, EKEY_ENTRY_CALLBACK PfnEKeyEntry, LPBYTE pbEKeyEntry, LPBYTE pbEKeyEnd)

{

size_t EntryLength = InHeader.EntryLength;

while((pbEKeyEntry + EntryLength) <= pbEKeyEnd)

{

// ENCODING for Starcraft II Beta

BREAK_ON_XKEY3(pbEKeyEntry, 0x8b, 0x0d, 0x9a);

if(!PfnEKeyEntry(hs, InHeader, pbEKeyEntry))

return ERROR_INDEX_PARSING_DONE;

pbEKeyEntry += EntryLength;

}

return ERROR_SUCCESS;

}

static DWORD CaptureIndexHeader_V1(CASC_INDEX_HEADER & InHeader, LPBYTE pbFileData, size_t cbFileData, DWORD BucketIndex)

{

PFILE_INDEX_HEADER_V1 pIndexHeader = (PFILE_INDEX_HEADER_V1)pbFileData;

LPBYTE pbKeyEntries;

LPBYTE pbFileEnd = pbFileData + cbFileData;

size_t cbKeyEntries;

DWORD HeaderHash;

// Check the available size. Note that the index file can be just a header.

if((pbFileData + sizeof(FILE_INDEX_HEADER_V1)) > pbFileEnd)

return ERROR_BAD_FORMAT;

if(pIndexHeader->IndexVersion != 0x05 || pIndexHeader->BucketIndex != (BYTE)BucketIndex || pIndexHeader->field_8 == 0)

return ERROR_BAD_FORMAT;

if(pIndexHeader->EncodedSizeLength != 0x04 || pIndexHeader->StorageOffsetLength != 0x05 || pIndexHeader->EKeyLength != 0x09)

return ERROR_NOT_SUPPORTED;

// Verify the header hash

HeaderHash = pIndexHeader->HeaderHash;

pIndexHeader->HeaderHash = 0;

if(hashlittle(pbFileData, sizeof(FILE_INDEX_HEADER_V1), 0) != HeaderHash)

return ERROR_BAD_FORMAT;

// Return the header hash back

pIndexHeader->HeaderHash = HeaderHash;

// Copy the fields

InHeader.IndexVersion = pIndexHeader->IndexVersion;

InHeader.BucketIndex = pIndexHeader->BucketIndex;

InHeader.StorageOffsetLength = pIndexHeader->StorageOffsetLength;

InHeader.EncodedSizeLength = pIndexHeader->EncodedSizeLength;

InHeader.EKeyLength = pIndexHeader->EKeyLength;

InHeader.FileOffsetBits = pIndexHeader->FileOffsetBits;

InHeader.Alignment = 0;

InHeader.SegmentSize = pIndexHeader->SegmentSize;

// Determine the size of the header

InHeader.HeaderLength = sizeof(FILE_INDEX_HEADER_V1);

InHeader.HeaderPadding = 0;

InHeader.EntryLength = pIndexHeader->EKeyLength + pIndexHeader->StorageOffsetLength + pIndexHeader->EncodedSizeLength;

InHeader.EKeyCount = pIndexHeader->EKeyCount1 + pIndexHeader->EKeyCount2;

// Verify the entries hash - 1st block

pbKeyEntries = pbFileData + InHeader.HeaderLength;

cbKeyEntries = pIndexHeader->EKeyCount1 * InHeader.EntryLength;

if((pbKeyEntries + cbKeyEntries) > pbFileEnd)

return ERROR_FILE_CORRUPT;

if(hashlittle(pbKeyEntries, cbKeyEntries, 0) != pIndexHeader->KeysHash1)

return ERROR_FILE_CORRUPT;

// Verify the entries hash - 2nd block

pbKeyEntries = pbKeyEntries + cbKeyEntries;

cbKeyEntries = pIndexHeader->EKeyCount2 * InHeader.EntryLength;

if((pbKeyEntries + cbKeyEntries) > pbFileEnd)

return ERROR_FILE_CORRUPT;

if(hashlittle(pbKeyEntries, cbKeyEntries, 0) != pIndexHeader->KeysHash2)

return ERROR_FILE_CORRUPT;

return ERROR_SUCCESS;

}

static DWORD CaptureIndexHeader_V2(CASC_INDEX_HEADER & InHeader, LPBYTE pbFileData, size_t cbFileData, DWORD BucketIndex)

{

PFILE_INDEX_HEADER_V2 pIndexHeader;

LPBYTE pbFileEnd = pbFileData + cbFileData;

// Check for guarded block

if((pbFileData = CaptureGuardedBlock1(pbFileData, pbFileEnd)) == NULL)

return ERROR_FILE_CORRUPT;

pIndexHeader = (PFILE_INDEX_HEADER_V2)pbFileData;

// Verify the content of the index header

if(pIndexHeader->IndexVersion != 0x07 || pIndexHeader->BucketIndex != (BYTE)BucketIndex || pIndexHeader->ExtraBytes != 0x00)

return ERROR_BAD_FORMAT;

if(pIndexHeader->EncodedSizeLength != 0x04 || pIndexHeader->StorageOffsetLength != 0x05 || pIndexHeader->EKeyLength != 0x09)

return ERROR_BAD_FORMAT;

// Capture the values from the index header

InHeader.IndexVersion = pIndexHeader->IndexVersion;

InHeader.BucketIndex = pIndexHeader->BucketIndex;

InHeader.StorageOffsetLength = pIndexHeader->StorageOffsetLength;

InHeader.EncodedSizeLength = pIndexHeader->EncodedSizeLength;

InHeader.EKeyLength = pIndexHeader->EKeyLength;

InHeader.FileOffsetBits = pIndexHeader->FileOffsetBits;

InHeader.Alignment = 0;

InHeader.SegmentSize = pIndexHeader->SegmentSize;

// Supply the lengths

InHeader.HeaderLength = sizeof(FILE_INDEX_GUARDED_BLOCK) + sizeof(FILE_INDEX_HEADER_V2);

InHeader.HeaderPadding = 8;

InHeader.EntryLength = pIndexHeader->EKeyLength + pIndexHeader->StorageOffsetLength + pIndexHeader->EncodedSizeLength;

InHeader.EKeyCount = 0;

return ERROR_SUCCESS;

}

static DWORD LoadIndexFile_V1(TCascStorage * hs, CASC_INDEX_HEADER & InHeader, EKEY_ENTRY_CALLBACK PfnEKeyEntry, LPBYTE pbFileData, size_t cbFileData)

{

LPBYTE pbEKeyEntries = pbFileData + InHeader.HeaderLength + InHeader.HeaderPadding;

// Remember the values from the index header

SaveFileOffsetBitsAndEKeyLength(hs, InHeader.FileOffsetBits, InHeader.EKeyLength);

// Load the entries from a continuous array

return LoadIndexItems(hs, InHeader, PfnEKeyEntry, pbEKeyEntries, pbFileData + cbFileData);

}

static DWORD LoadIndexFile_V2(TCascStorage * hs, CASC_INDEX_HEADER & InHeader, EKEY_ENTRY_CALLBACK PfnEKeyEntry, LPBYTE pbFileData, size_t cbFileData)

{

LPBYTE pbEKeyEntry;

LPBYTE pbFileEnd = pbFileData + cbFileData;

LPBYTE pbFilePtr = pbFileData + InHeader.HeaderLength + InHeader.HeaderPadding;

size_t EKeyEntriesLength;

DWORD BlockSize = 0;

DWORD dwErrCode = ERROR_NOT_SUPPORTED;

// Remember the values from the index header

SaveFileOffsetBitsAndEKeyLength(hs, InHeader.FileOffsetBits, InHeader.EKeyLength);

// Get the pointer to the first block of EKey entries

if((pbEKeyEntry = CaptureGuardedBlock2(pbFilePtr, pbFileEnd, InHeader.EntryLength, &BlockSize)) != NULL)

{

// Supply the number of EKey entries

InHeader.HeaderPadding += sizeof(FILE_INDEX_GUARDED_BLOCK);

// Load the continuous array of EKeys

return LoadIndexItems(hs, InHeader, PfnEKeyEntry, pbEKeyEntry, pbEKeyEntry + BlockSize);

}

// Get the pointer to the second block of EKey entries.

// They are alway at the position aligned to 4096

EKeyEntriesLength = pbFileEnd - pbFilePtr;

if(EKeyEntriesLength >= 0x7800)

{

LPBYTE pbStartPage = pbFileData + 0x1000;

LPBYTE pbEndPage = pbStartPage + FILE_INDEX_PAGE_SIZE;

size_t AlignedLength = ALIGN_TO_SIZE(InHeader.EntryLength, 4);

// Parse the chunks with the EKey entries

while(pbStartPage < pbFileEnd)

{

pbEKeyEntry = pbStartPage;

while(pbEKeyEntry < pbEndPage)

{

// Check the EKey entry protected by 32-bit hash

if((pbEKeyEntry = CaptureGuardedBlock3(pbEKeyEntry, pbEndPage, InHeader.EntryLength)) == NULL)

break;

// CASC\\0001: Encoding

//BREAK_ON_XKEY3(pbEKeyEntry, 0xbc, 0xe8, 0x23);

// Call the EKey entry callback

if(!PfnEKeyEntry(hs, InHeader, pbEKeyEntry))

return ERROR_INDEX_PARSING_DONE;

pbEKeyEntry += AlignedLength;

}

// Move to the next chunk

pbStartPage += FILE_INDEX_PAGE_SIZE;

}

dwErrCode = ERROR_SUCCESS;

}

return dwErrCode;

}

static DWORD LoadIndexFile(TCascStorage * hs, EKEY_ENTRY_CALLBACK PfnEKeyEntry, LPBYTE pbFileData, size_t cbFileData, DWORD BucketIndex)

{

CASC_INDEX_HEADER InHeader;

// Check for CASC version 2

if(CaptureIndexHeader_V2(InHeader, pbFileData, cbFileData, BucketIndex) == ERROR_SUCCESS)

return LoadIndexFile_V2(hs, InHeader, PfnEKeyEntry, pbFileData, cbFileData);

// Check for CASC index version 1

if(CaptureIndexHeader_V1(InHeader, pbFileData, cbFileData, BucketIndex) == ERROR_SUCCESS)

return LoadIndexFile_V1(hs, InHeader, PfnEKeyEntry, pbFileData, cbFileData);

// Should never happen

assert(false);

return ERROR_BAD_FORMAT;

}

// Checks the EKey entry for EKey of the ENCODING manifest

static bool InsertEncodingEKeyToMap(TCascStorage * hs, CASC_INDEX_HEADER &, LPBYTE pbEKeyEntry)

{

hs->IndexEKeyMap.InsertObject(pbEKeyEntry, pbEKeyEntry);

return true;

}

static DWORD ProcessLocalIndexFiles(TCascStorage * hs, EKEY_ENTRY_CALLBACK PfnEKeyEntry, DWORD dwIndexCount)

{

DWORD dwErrCode = ERROR_SUCCESS;

// Load each index file

for(DWORD i = 0; i < dwIndexCount; i++)

{

CASC_INDEX & IndexFile = hs->IndexFiles[i];

// Inform the user about what we are doing

if(InvokeProgressCallback(hs, "Loading index files", NULL, i, dwIndexCount))

{

dwErrCode = ERROR_CANCELLED;

break;

}

// Load the index file

if((dwErrCode = LoadIndexFile(hs, PfnEKeyEntry, IndexFile.pbFileData, IndexFile.cbFileData, i)) != ERROR_SUCCESS)

break;

}

// Swallow the "done parsing" error

if(dwErrCode == ERROR_INDEX_PARSING_DONE)

dwErrCode = ERROR_SUCCESS;

// Remember the number of files that are present locally

hs->LocalFiles = hs->CKeyArray.ItemCount();

return dwErrCode;

}

static DWORD LoadLocalIndexFiles(TCascStorage * hs)

{

ULONGLONG TotalSize = 0;

DWORD dwIndexCount = 0;

DWORD dwErrCode;

// Inform the user about what we are doing

if(InvokeProgressCallback(hs, "Loading index files", NULL, 0, 0))

return ERROR_CANCELLED;

// Perform the directory scan

if((dwErrCode = ScanIndexDirectory(hs->szIndexPath, IndexDirectory_OnFileFound, hs)) == ERROR_SUCCESS)

{

// If no index file was found, we cannot load anything

if(hs->szIndexFormat == NULL)

return ERROR_FILE_NOT_FOUND;

// Load each index file

for(DWORD i = 0; i < CASC_INDEX_COUNT; i++)

{

CASC_INDEX & IndexFile = hs->IndexFiles[i];

DWORD cbFileData = 0;

// Create the file name

if((IndexFile.szFileName = CreateIndexFileName(hs, i, IndexFile.NewSubIndex)) == NULL)

return ERROR_NOT_ENOUGH_MEMORY;

// WoW6 actually reads THE ENTIRE file to memory. Verified on Mac build (x64).

if((IndexFile.pbFileData = LoadFileToMemory(IndexFile.szFileName, &cbFileData)) == NULL)

{

// Storages downloaded by Blizzget tool don't have all index files present

if((dwErrCode = GetCascError()) == ERROR_FILE_NOT_FOUND)

{

dwErrCode = ERROR_SUCCESS;

break;

}

return dwErrCode;

}

// Add to the total size of the index files

IndexFile.cbFileData = cbFileData;

TotalSize += cbFileData;

dwIndexCount++;

}

// Build the map of EKey -> IndexEKeyEntry

dwErrCode = hs->IndexEKeyMap.Create((size_t)(TotalSize / sizeof(FILE_EKEY_ENTRY)), CASC_EKEY_SIZE, 0);

if(dwErrCode == ERROR_SUCCESS)

{

dwErrCode = ProcessLocalIndexFiles(hs, InsertEncodingEKeyToMap, dwIndexCount);

}

}

return dwErrCode;

}

//-----------------------------------------------------------------------------

// Online index files

// https://wowdev.wiki/TACT#CDN_File_Organization

static DWORD CaptureArcIndexFooter(CASC_ARCINDEX_FOOTER & InFooter, LPBYTE pbIndexFile, DWORD cbIndexFile)

{

FILE_INDEX_FOOTER<0x08> * pFooter08;

BYTE checksum_data[0x40] = { 0 };

BYTE md5_hash[MD5_HASH_SIZE];

DWORD checksum_data_length;

// Clear the entire structure

memset(&InFooter, 0, sizeof(CASC_ARCINDEX_FOOTER));

// Check the variant for checksum == 0x08

pFooter08 = (FILE_INDEX_FOOTER<0x08> *)(pbIndexFile + cbIndexFile - sizeof(FILE_INDEX_FOOTER<0x08>));

if (pFooter08->Version == 1 && pFooter08->Reserved[0] == 0 && pFooter08->Reserved[1] == 0 && pFooter08->FooterHashBytes == 8)

{

// Copy the entire structure

memcpy(InFooter.TocHash, pFooter08->TocHash, MD5_HASH_SIZE);

memcpy(InFooter.FooterHash, pFooter08->FooterHash, pFooter08->FooterHashBytes);

InFooter.Version = pFooter08->Version;

InFooter.OffsetBytes = pFooter08->OffsetBytes;

InFooter.SizeBytes = pFooter08->SizeBytes;

InFooter.EKeyLength = pFooter08->EKeyLength;

InFooter.FooterHashBytes = pFooter08->FooterHashBytes;

InFooter.PageLength = pFooter08->PageSizeKB << 10;

InFooter.ItemLength = pFooter08->EKeyLength + pFooter08->OffsetBytes + pFooter08->SizeBytes;

InFooter.FooterLength = sizeof(FILE_INDEX_FOOTER<0x08>);

InFooter.ElementCount = ConvertBytesToInteger_4_LE(pFooter08->ElementCount);

// Verify the hash. FooterHash needs to be cleared in order to calculate footer hash properly

checksum_data_length = FIELD_OFFSET(FILE_INDEX_FOOTER<0x08>, FooterHash) - FIELD_OFFSET(FILE_INDEX_FOOTER<0x08>, Version);

memcpy(checksum_data, &pFooter08->Version, checksum_data_length);

CascCalculateDataBlockHash(checksum_data, sizeof(FILE_INDEX_FOOTER<0x08>) - MD5_HASH_SIZE, md5_hash);

if(!memcmp(md5_hash, InFooter.FooterHash, InFooter.FooterHashBytes))

return ERROR_SUCCESS;

}

assert(false);

return ERROR_BAD_FORMAT;

}

static DWORD CaptureIndexEntry(CASC_ARCINDEX_FOOTER & InFooter, CASC_EKEY_ENTRY & EKeyEntry, LPBYTE pbIndexPage, LPBYTE pbIndexPageEnd, size_t nArchive)

{

ULONGLONG StorageOffset = nArchive;

ULONGLONG ArchiveOffset;

// If there enough bytes for one entry/

if ((pbIndexPage + InFooter.ItemLength) > pbIndexPageEnd)

return ERROR_BAD_FORMAT;

// Capture the EKey (variable length)

pbIndexPage = CaptureEncodedKey(EKeyEntry.EKey, pbIndexPage, InFooter.EKeyLength);

// Copy the archive offset

ArchiveOffset = ConvertBytesToInteger_X(pbIndexPage + InFooter.SizeBytes, InFooter.OffsetBytes);

if (ArchiveOffset >= 0x10000000)

return ERROR_BAD_FORMAT;

// Capture the storage offset and encoded size

EKeyEntry.StorageOffset = (StorageOffset << (InFooter.OffsetBytes * 8)) | ArchiveOffset;

EKeyEntry.EncodedSize = ConvertBytesToInteger_X(pbIndexPage, InFooter.SizeBytes);

EKeyEntry.Alignment = 0;

// Is there a valid hash?

return CascIsValidMD5(EKeyEntry.EKey) ? ERROR_SUCCESS : ERROR_BAD_FORMAT;

}

static DWORD VerifyIndexSize(CASC_ARCINDEX_FOOTER & InFooter, LPBYTE pbIndexFile, size_t cbIndexFile, LPBYTE * PtrIndexEnd)

{

size_t nPageCount;

// Set the new length (without the footer)

cbIndexFile = cbIndexFile - InFooter.FooterLength;

nPageCount = cbIndexFile / (InFooter.PageLength + MD5_HASH_SIZE);

// There must be equal or more pages

if(((InFooter.PageLength + MD5_HASH_SIZE) * nPageCount) > cbIndexFile)

return ERROR_BAD_FORMAT;

// Return the end-of-index

nPageCount = cbIndexFile / (InFooter.PageLength + MD5_HASH_SIZE);

PtrIndexEnd[0] = pbIndexFile + (nPageCount * InFooter.PageLength);

return ERROR_SUCCESS;

}

static DWORD LoadArchiveIndexPage(TCascStorage * hs, CASC_ARCINDEX_FOOTER & InFooter, LPBYTE pbIndexPage, LPBYTE pbIndexPageEnd, size_t nArchive)

{

CASC_EKEY_ENTRY EKeyEntry;

DWORD dwErrCode;

while (pbIndexPage <= pbIndexPageEnd)

{

// Capture the index entry

dwErrCode = CaptureIndexEntry(InFooter, EKeyEntry, pbIndexPage, pbIndexPageEnd, nArchive);

if (dwErrCode != ERROR_SUCCESS)

break;

// Insert a new entry to the index array

if((hs->IndexArray.Insert(&EKeyEntry, 1)) == NULL)

return ERROR_NOT_ENOUGH_MEMORY;

// Move to the next entry

pbIndexPage += InFooter.ItemLength;

}

return ERROR_SUCCESS;

}

static DWORD LoadArchiveIndexFile(TCascStorage * hs, LPBYTE pbIndexFile, DWORD cbIndexFile, size_t nArchive)

{

CASC_ARCINDEX_FOOTER InFooter;

LPBYTE pbIndexEnd = NULL;

DWORD dwErrCode;

// Validate and capture the footer

dwErrCode = CaptureArcIndexFooter(InFooter, pbIndexFile, cbIndexFile);

if (dwErrCode != ERROR_SUCCESS)

return dwErrCode;

// Remember the file offset and EKey length

SaveFileOffsetBitsAndEKeyLength(hs, InFooter.OffsetBytes * 8, InFooter.EKeyLength);

// Verify the size of the index file

dwErrCode = VerifyIndexSize(InFooter, pbIndexFile, cbIndexFile, &pbIndexEnd);

if (dwErrCode != ERROR_SUCCESS)

return dwErrCode;

// Parse all pages

while (pbIndexFile < pbIndexEnd)

{

// Load the entire page

dwErrCode = LoadArchiveIndexPage(hs, InFooter, pbIndexFile, pbIndexFile + InFooter.PageLength, nArchive);

if (dwErrCode != ERROR_SUCCESS)

break;

// Move to the next page

pbIndexFile += InFooter.PageLength;

}

return ERROR_SUCCESS;

}

static DWORD BuildMapOfArchiveIndices(TCascStorage * hs)

{

PCASC_EKEY_ENTRY pEKeyEntry;

size_t nItemCount = hs->IndexArray.ItemCount();

DWORD dwErrCode;

// Create the map

dwErrCode = hs->IndexMap.Create(nItemCount, MD5_HASH_SIZE, FIELD_OFFSET(CASC_EKEY_ENTRY, EKey));

if (dwErrCode != ERROR_SUCCESS)

return dwErrCode;

// Insert all items

for(size_t i = 0; i < nItemCount; i++)

{

pEKeyEntry = (PCASC_EKEY_ENTRY)hs->IndexArray.ItemAt(i);

if (pEKeyEntry != NULL)

{

if (!hs->IndexMap.InsertObject(pEKeyEntry, pEKeyEntry->EKey))

{

return ERROR_NOT_ENOUGH_MEMORY;

}

}

}

return dwErrCode;

}

static DWORD LoadArchiveIndexFiles(TCascStorage * hs)

{

LPBYTE pbFileData;

TCHAR szLocalPath[MAX_PATH];

DWORD cbFileData = 0;

size_t nArchiveCount = (hs->ArchivesKey.cbData / MD5_HASH_SIZE);

DWORD dwErrCode = ERROR_SUCCESS;

// Create the array object for the indices

dwErrCode = hs->IndexArray.Create(sizeof(CASC_EKEY_ENTRY), 0x10000);

if (dwErrCode != ERROR_SUCCESS)

return dwErrCode;

// Load all the indices

for (size_t i = 0; i < nArchiveCount; i++)

{

CASC_CDN_DOWNLOAD CdnsInfo = {0};

LPBYTE pbIndexHash = hs->ArchivesKey.pbData + (i * MD5_HASH_SIZE);

// Inform the user about what we are doing

if(InvokeProgressCallback(hs, "Downloading archive indexes", NULL, (DWORD)(i), (DWORD)(nArchiveCount)))

{

dwErrCode = ERROR_CANCELLED;

break;

}

// Prepare the download structure for "%CDNS_HOST%/%CDNS_PATH%/##/##/EKey" file

CdnsInfo.szCdnsPath = hs->szCdnPath;

CdnsInfo.szPathType = _T("data");

CdnsInfo.pbEKey = pbIndexHash;

CdnsInfo.szExtension = _T(".index");

CdnsInfo.szLocalPath = szLocalPath;

CdnsInfo.ccLocalPath = _countof(szLocalPath);

dwErrCode = DownloadFileFromCDN(hs, CdnsInfo);

// Load and parse the archive index

if (dwErrCode == ERROR_SUCCESS)

{

// Load the index file to memory

pbFileData = LoadFileToMemory(szLocalPath, &cbFileData);

if (pbFileData && cbFileData)

{

dwErrCode = LoadArchiveIndexFile(hs, pbFileData, cbFileData, i);

CASC_FREE(pbFileData);

}

}

// Break if an error

if (dwErrCode != ERROR_SUCCESS)

break;

}

// Build map of EKey -> CASC_EKEY_ENTRY

if (dwErrCode == ERROR_SUCCESS)

dwErrCode = BuildMapOfArchiveIndices(hs);

return dwErrCode;

}

//-----------------------------------------------------------------------------

// Public functions

bool CopyEKeyEntry(TCascStorage * hs, PCASC_CKEY_ENTRY pCKeyEntry)

{

LPBYTE pbEKeyEntry = (LPBYTE)hs->IndexEKeyMap.FindObject(pCKeyEntry->EKey);

// Don't do this on online storages

if(!(hs->dwFeatures & CASC_FEATURE_ONLINE))

{

// If the file was found, then copy the content to the CKey entry

pbEKeyEntry = (LPBYTE)hs->IndexEKeyMap.FindObject(pCKeyEntry->EKey);

if(pbEKeyEntry == NULL)

return false;

pCKeyEntry->StorageOffset = ConvertBytesToInteger_5(pbEKeyEntry + hs->EKeyLength);

pCKeyEntry->EncodedSize = ConvertBytesToInteger_4_LE(pbEKeyEntry + hs->EKeyLength + 5);

pCKeyEntry->Flags |= CASC_CE_FILE_IS_LOCAL;

}

return true;

}

DWORD LoadIndexFiles(TCascStorage * hs)

{

// For local storages, load the index files from the disk

// For online storages, load the index files from the cache / internet

if(hs->dwFeatures & CASC_FEATURE_ONLINE)

{

return LoadArchiveIndexFiles(hs);

}

else

{

return LoadLocalIndexFiles(hs);

}

}

void FreeIndexFiles(TCascStorage * hs)

{

// Free the map of EKey -> Index Ekey item

hs->IndexEKeyMap.Free();

// Free all loaded index files

for(size_t i = 0; i < CASC_INDEX_COUNT; i++)

{

CASC_INDEX & IndexFile = hs->IndexFiles[i];

// Free the file data

CASC_FREE(IndexFile.pbFileData);

IndexFile.cbFileData = 0;

// Free the file name

CASC_FREE(IndexFile.szFileName);

}

}