Extract and view Black Ops 4 files - textures, icons etc

[havhog]

Teach me please

Anybody who could make a step by step tutor here for a complete noob in file extraction? I got Wraith Acron

[aluigi]

Please read the rules for understanding what information/file you have to provide when opening a topic:
viewtopic.php?f=21&t=4

Black Ops 3 used the KAPI/IPAK format, maybe this new game uses the same, try this script:
http://aluigi.org/bms/cod_kapi.bms

[lolbas]

Black Ops 3 used the KAPI/IPAK format, maybe this new game uses the same, try this script:
http://aluigi.org/bms/cod_kapi.bms

Yes, it does. File version is 11 and this script is not handling it properly.
Here are some samples: http://www.mediafire.com/file/h8kmr3fna ... k.zip/file

And here are a few other things I noticed:

- Every file with KAPI header contained 4 definitions of file data:

1. Files data (encrypted/compressed/whatnot)
2. Some meta-data(?) probably. 3x u64, first one is UUID, the other two I couldn't figure out.
3. File indices (from what I noticed - list of 8 byte UUIDs)
4. Text description of files

- Most of the files have unknown chunk flag 0x8
- Text description of files is file UUID + string of meta-data

I've wrote a script to extract files but the issue is most of files have unknown flag 0x8, other flags from your cod_kapi.bms also occur.

Code: Select all

idstring "KAPI"
get ZERO short
get VER short

if VER > 10
    get DUMMY longlong
    get KAPI_SIZE longlong
    
    #for SECTION = 0
        get FILES longlong
        get OFFSET longlong
        get SIZE longlong
        
        if OFFSET == 0 || OFFSET u> KAPI_SIZE
            break
        endif
        
        savepos SECTION_OFF
        goto OFFSET
        
        for i = 0 < FILES
            savepos OFFSET
            get CHUNKS long
            get DUMMY long
            
            if CHUNKS u< 0xffff
                if CHUNKS == 0xa7a7a7a7
                    break
                endif
                
                set FILESIZE long 0
                
                for c = 0 < CHUNKS
                    get CHUNK_META long
                    
                    xmath FILESIZE "FILESIZE + (CHUNK_META & 0x00FFFFFF)"
                next c
                
                padding 128
                savepos CHUNKS_OFFSET
                print "Offset %CHUNKS_OFFSET|X%"
                log "" CHUNKS_OFFSET FILESIZE
                
                goto FILESIZE 0 SEEK_CUR
                padding 128
            endif
        next i

        goto SECTION_OFF
    #next SECTION

endif

010Editor template that parses entire file:

Code: Select all

//------------------------------------------------
//--- 010 Editor v9.0 Binary Template
//
//      File: 
//   Authors: 
//   Version: 
//   Purpose: 
//  Category: 
// File Mask: 
//  ID Bytes: 
//   History: 
//------------------------------------------------

local char KAPI_HEADER[4] = "KAPI";

uint Normalize(uint value) {
    return Ceil(value / 128.0) * 128;
}

struct KapiFile {
    struct KapiFileData (uint chunks) {
        local int c;
        local uint chunk_size;
        local uint filesize = 0;
        local int chunk_flags;
        
        for (c = 0; c < chunks; c++) {
            uint chunk_meta <bgcolor=0x00FF00, name="Chunk Meta">;
            chunk_size = chunk_meta & 0x00FFFFFF;
            chunk_flags = (chunk_meta & 0xFF000000) >> 24;
            Assert(chunk_flags == 0 || chunk_flags == 8 || chunk_flags == 0xCF, "Chunk flag is not 0, 8 or 0xCF!");
            filesize += chunk_size;
        }
        
        FSeek(foffset + 0x80);
        
        byte contents[Normalize(filesize)] <bgcolor=0x00FF00, name="Contents">;
    };
    
    local uint foffset = FTell();
    uint64 chunks <bgcolor=0x00FF00, name="Chunks Count">;
    
    if (chunks < 0xFFFF) {
        if (chunks == 0xA7A7A7A7A7A7A7A7) {
            break;
        }
        KapiFileData filedata(chunks) <name="File Data">;
    }
};

struct KapiFiles(uint64 files_count) {
    local uint i;
    for (i = 0; i < files_count; i++) {
        KapiFile file <name="File">;
    }
};

struct KapiMeta(uint64 files_count) {
    struct KapiFileMeta {
        uint64 id <bgcolor=0x00FF00, name="ID">;
        uint64 no_idea1 <bgcolor=0x0000FF>;
        uint64 no_idea2 <bgcolor=0x0000FF>;
    };
    
    local uint i;
    for (i = 0; i < files_count; i++) {
        KapiFileMeta meta <name="Meta">;
    }
};

struct KapiIndices(uint64 files_count) {
    // files_count can be many times bigger than in KapiFiles
    local uint i;
    for (i = 0; i < files_count; i++) {
        uint64 id <bgcolor=0x00FF00, name="ID">;
    }
};

struct KapiText(uint64 files_count) {
    struct KapiFileDesc {
        uint64 id <bgcolor=0x00FF00, name="ID">;
        uint64 text_length <bgcolor=0x00FF00, name="Text Length">;
        char text[text_length] <bgcolor=0x00FF00>;
    };
    local uint i;
    for (i = 0; i < files_count; i++) {
        KapiFileDesc file_desc <bgcolor=0x00FF00, name="File Desc">;
    }
};

struct KapiBlock(int type) {
    uint64 files_count <bgcolor=0x00FF00, name="Files Count">;
    uint64 offset <bgcolor=0x00FF00, name="Offset">;
    uint64 size <bgcolor=0x00FF00, name="Data Size">;
    
    if (offset == 0 || size == 0 || offset > kapisize) {
        break;
    }
    
    local uint fpos = FTell();
    FSeek(offset);
    
    switch (type) {
        case 1: KapiFiles files(files_count); break;
        case 2: KapiMeta meta(files_count); break;
        case 3: KapiIndices indices(files_count); break;
        case 4: KapiText text_index(files_count); break;
    }
    
    FSeek(fpos);
};

struct KapiData {
    uint64 files <bgcolor=0x00FF00, name="Files Count">;
    uint64 offset <bgcolor=0x00FF00, name="Offset">;
    uint64 size <bgcolor=0x00FF00, name="Data Size">;
};

struct KAPI11 {
    uint64 dummy <bgcolor=0x0000FF>;
    uint64 kapisize <bgcolor=0x00FF00>;
    
    KapiBlock block(1);
    KapiBlock block(2);
    KapiBlock block(3);
    KapiBlock block(4);
};

struct {
    uchar signature[4] <bgcolor=0x00FF00, name="Signature">;
    Assert(signature == KAPI_HEADER, "Wrong file format!");
    ushort zero <bgcolor=0x00FF00>;
    ushort version <bgcolor=0x00FF00, name="Version">;
    
    if (version > 10) {
        KAPI11 kapi;
    }
} file <name="KAPI">;

It would be great if you could look into this flag!

[aluigi]

I updated the script few hours ago:
http://aluigi.org/bms/cod_kapi.bms

[lolbas]

few hours ago

Wow, that's unexpected surprise! Thanks, I will take a look into that!

Not familiar with oodle compression, but I noticed that files created by your script miss first 12 bytes from their original content while leaving next 8 unchanged. Is that the expected decompression result?

[aluigi]

The script simply decompresses the chunks in sequential order so there is no space for corrupting or cutting parts of the extracted files.
Additionaly oodle is one of those "picky" algorithms that returns errors if the provided data, compressed and decompressed size isn't correct.

[lolbas]

Thank you again for the script. I've used it as a base to change generated files.

Script follows strict pattern I noticed across files, hence allowing me to do certain stuff, particularly I create files with their UID as name. I am also able to save text data per file (the huge load of text at the file end) but unfortunately its count can be way bigger than the files xpak contains. Does QuickBMS supports some kind of builtin array search? I wanted to extract text meta only for existing files. 325MB file contains 48k meta files...

You may want to check my script and maybe add it to index as alternative to yours!

Code: Select all

idstring "KAPI"
get ZERO short
get VER short

###############################
# Arrays:
#   0 - UIDs of files in KAPI
#   1 - Per file: chunk size
#   2 - Per file: chunk flags
###############################

if VER > 10
    get DUMMY longlong
    get KAPI_SIZE longlong
    
    get DATA_FILES_COUNT longlong
    get DATA_OFFSET longlong
    get DATA_SIZE longlong
    
    get META_FILES_COUNT longlong
    get META_OFFSET longlong
    get META_SIZE longlong
    
    get INDEX_FILES_COUNT longlong
    get INDEX_OFFSET longlong
    get INDEX_SIZE longlong
    
    get TEXT_FILES_COUNT longlong
    get TEXT_OFFSET longlong
    get TEXT_SIZE longlong
    
    if DATA_FILES_COUNT <> META_FILES_COUNT
        || DATA_OFFSET u> KAPI_SIZE
        || META_OFFSET u> KAPI_SIZE
        || INDEX_OFFSET u> KAPI_SIZE
        || TEXT_OFFSET u> KAPI_SIZE
        print "Error: data files count is not equal to meta files count - can't match files with their UIDs"
        cleanexit
    endif
    
    ########################################################
    # First we need to get list of file UIDs for later use #
    ########################################################
    
    if META_OFFSET != 0
        goto META_OFFSET
        
        for i = 0 < META_FILES_COUNT
            get UID1 long
            get UID2 long
            
            get UNK1 longlong
            get UNK2 longlong
            
            string SUID1 p= "%08X" UID1
            string UID p= "%08X" UID2
            
            string UID + SUID1
            
            putarray 0 i UID
        next i
    endif
    
    ###########################################################################
    # Now that we have them, we can use UIDs to save files under these names. #
    ###########################################################################
    
    if DATA_OFFSET != 0
        goto DATA_OFFSET
        
        for i = 0 < DATA_FILES_COUNT
            get CHUNKS long
            get DUMMY long
            
            if CHUNKS u< 0xffff
                if CHUNKS == 0xa7a7a7a7
                    break
                endif
                
                set FILESIZE long 0
                
                for c = 0 < CHUNKS
                    get CHUNK_META long
                    
                    math CHUNK_SIZE = CHUNK_META
                    math CHUNK_SIZE & 0x00FFFFFF
                    
                    math CHUNK_FLAGS = CHUNK_META
                    math CHUNK_FLAGS & 0xFF000000
                    math CHUNK_FLAGS u>> 24
                    
                    math FILESIZE + CHUNK_SIZE
                    
                    putarray 1 c CHUNK_SIZE
                    putarray 2 c CHUNK_FLAGS
                next c
                
                padding 128
                savepos CHUNKS_OFFSET
                
                putvarchr MEMORY_FILE FILESIZE 0
                log MEMORY_FILE 0 0
                
                #for c = 0 < CHUNKS
                    #getarray CHUNK_SIZE 1 0
                    getarray CHUNK_FLAGS 2 0
                    
                    if CHUNK_FLAGS == 0
                        log MEMORY_FILE CHUNKS_OFFSET FILESIZE
                    elif CHUNK_FLAGS == 1
                        comtype lzolx
                        clog MEMORY_FILE CHUNKS_OFFSET FILESIZE 0x8000
                    elif CHUNK_FLAGS == 3
                        comtype lz4
                        clog MEMORY_FILE CHUNKS_OFFSET FILESIZE 0x8000
                    elif CHUNK_FLAGS == 8
                        comtype oodle
                        goto CHUNKS_OFFSET
                        get CHUNK_XSIZE long
                        math CHUNKS_OFFSET + 4
                        math FILESIZE - 4
                        clog MEMORY_FILE CHUNKS_OFFSET FILESIZE CHUNK_XSIZE
                    elif FLAGS == 0xcf
                        log MEMORY_FILE CHUNKS_OFFSET FILESIZE
                    else
                        print "Error: unknown CHUNK_FLAGS %CHUNK_FLAGS|X%, contact me"
                        cleanexit
                    endif
                    
                    #math CHUNKS_OFFSET + CHUNK_SIZE
                #next c
                
                getarray FILENAME 0 i
                get MEM_SIZE asize MEMORY_FILE
                
                log FILENAME 0 MEM_SIZE MEMORY_FILE
                
                goto FILESIZE 0 SEEK_CUR
                padding 128
            endif
        next i
    endif
    
    #####################################################
    # And finally we can dump meta description per file #
    #####################################################
    /*
    if TEXT_OFFSET != 0
        goto TEXT_OFFSET
        
        for i = 0 < TEXT_FILES_COUNT
            get UID1 long
            get UID2 long
            
            string SUID1 p= "%08X" UID1
            string UID p= "%08X" UID2
            
            string UID + SUID1
            string UID + ".meta"
            
            get TEXT_LENGTH longlong
            
            savepos OFFSET
            
            //print "%UID% %OFFSET|X% %TEXT_LENGTH%"
            log UID OFFSET TEXT_LENGTH
            
            math OFFSET + TEXT_LENGTH
            goto OFFSET
        next i
    endif
    */
endif

[aluigi]

Array searching is not available.
There is a simple trick for using "whitelists" like that, for example:

Code: Select all

set WHITELIST string ",meta1,meta2,meta3,"
string MYMETA p ",%s," MYMETA # MYMETA was "meta2"
if WHITELIST & MYMETA
   do stuff
endif

The second big "if" in your script may be not handled correctly by quickbms, try to keep the command on one line and not containing many switches (quickbms supports max 32 arguments per command, if arg1 ... arg31)

[lolbas]

/Oh okay, thank you

There is a simple trick for using "whitelists" like that

I doubt it suits as whitelist is not known beforehand and it can contain 100-200000 file UIDs(for 20GB data file).

Any chance you could look deeper into the file? Chunk size limit is about 262112 bytes and weirdly enough, no files larger than this are extracted with the tool (well, I tested just a few xpaks), except for the 0xCD files. (Seems like) Every xpak file is followed by *_d.xpak, *.fd and *.ff, maybe they are related to decoding this... I also find it strange that files count is mostly much smaller than indices and text meta count (6k vs 47k for example) and usually block 1 count = block 2 count and block 3 count = block 4 count. What I also noticed is that 20GB file (base.xpak) (it actually is split up into 22 files with a maximum size of 996147200 bytes (950MB) and in this files block 3 (raw indices list) is empty and block 1 count = block 2 count = block 4 count = 191960 for this file. I could upload more samples, but don't really now which ones would be better

[aluigi]

Honestly I don't think I can work further on this format, at least not at the moment.

[lolbas]

Sure, no problems

I actually noticed that Greyhound was recently updated and can extract xpak files (images for sure, even converts them to png, not sure about models and other): https://github.com/Scobalula/Greyhound