Unbpe source code

Programming related discussions related to game research
eatrawmeat391
Posts: 9
Joined: Sun Jul 17, 2016 5:23 am

Unbpe source code

Post by eatrawmeat391 »

So I got the unbpe source code.Problem is,I only know C a little.What parameter do you have to call in the main function to let you decompress a bpe string?I will not use QuickBMS script because my python program needs to be able to decompress data from memory each iteration

Code: Select all

/*
  by Luigi Auriemma
reversed from asmodean's unrrbpe.exe
*/

#include <string.h>

static int xgetc(unsigned char **in, unsigned char *inl) {
    int     ret;
    if(*in >= inl) return(-1);
    ret = **in;
    (*in)++;
    return(ret);
}

int yuke_bpe(unsigned char *in, int insz, unsigned char *out, int outsz, int fill_outsz) {
    unsigned char   stack[512 + 4096];
    int             c,
                    count,
                    i,
                    size,
                    n;

    unsigned char   *inl,
                    *o,
                    *outl;

    inl  = in + insz;
    o    = out;
    outl = out + outsz;

    count = 0;
    for(;;) {
        i = 0;
        do {
            if((c = xgetc(&in, inl)) < 0) break;
            if(c > 127) {
                c -= 127;
                while((c > 0) && (i < 256)) {
                    stack[i * 2] = i;
                    c--;
                    i++;
                }
            }
            c++;
            while((c > 0) && (i < 256)) {
                if((n = xgetc(&in, inl)) < 0) break;
                stack[i * 2] = n;
                if(i != n) {
                    if((n = xgetc(&in, inl)) < 0) break;
                    stack[(i * 2) + 1] = n;
                }
                c--;
                i++;
            }
        } while(i < 256);

        if((n = xgetc(&in, inl)) < 0) break;
        size = n;
        if((n = xgetc(&in, inl)) < 0) break;
        size |= (n << 8);

        while(size || count) {
            if(count) {
                count--;
                n = stack[count + 512];
            } else {
                if((n = xgetc(&in, inl)) < 0) break;
                size--;
            }
            c = stack[n * 2];
            if(n == c) {
                if(o >= outl) return(-1);
                *o++ = n;
            } else {
                if((count + 512 + 2) > sizeof(stack)) return(-1);
                stack[count + 512] = stack[(n * 2) + 1];
                stack[count + 512 + 1] = c;
                count += 2;
            }
        }
    }
    if(fill_outsz) {    // this is what is wanted by the format
        memset(o, 0, outl - o);
        o = outl;
    }
    return(o - out);
}
aluigi
Site Admin
Posts: 12984
Joined: Wed Jul 30, 2014 9:32 pm

Re: Unbpe source code

Post by aluigi »

That yuke_bpe is just a function without main, it takes 5 arguments:
  • input buffer
  • size of input buffer
  • output buffer
  • size of the output buffer
  • set it to one
You can compile it as library and use it directly inside python.
Google keywords: calling c functions from python
eatrawmeat391
Posts: 9
Joined: Sun Jul 17, 2016 5:23 am

Re: Unbpe source code

Post by eatrawmeat391 »

I have implemented code for the main function.This program will take 3 parameters: 'bpe_string bpe_size uncompressed_size'.For example 'unbpe_mem.exe [bpe_string] [bpe_size] [uncompressed_size]'.I use python program to call the C program
Before sending the bpe_string argument,my python program will do the following thing to the string:(I think these replaced string will never be present in a compressed string because of redundancies)
- Replace all '\x00' to all '\n\n\n' (Null byte)
- Replace all "\'" to all '\r\r\r' (Single quote)
- Replace all '\"' to all '\t\t\t' (Double quote)
- Replace all spaces with all '\a\a\a'
This will make the bpe_string 'valid' to use in Python subprocess module(which do the cmd calling).Then after the C program opens up it will decode the replaced character back.After that it calls yuke_bpe to try to decompress the string.If the program crashed then the BPE string is invalid,if it worked then called quickbms to generate an unpacked bpe file,comparing CRC32 checksum of the unpacked file to the original uncompressed file.

Code: Select all


/*
  by Luigi Auriemma
reversed from asmodean's unrrbpe.exe
*/
#include <string.h>
#include <stdlib.h>

static int xgetc(unsigned char **in, unsigned char *inl) {
    int     ret;
    if(*in >= inl) return(-1);
    ret = **in;
    (*in)++;
    return(ret);
}

int yuke_bpe(unsigned char *in, int insz, unsigned char *out, int outsz, int fill_outsz) {
    unsigned char   stack[512 + 4096];
    int             c,
                    count,
                    i,
                    size,
                    n;

    unsigned char   *inl,
                    *o,
                    *outl;

    inl  = in + insz;
    o    = out;
    outl = out + outsz;

    count = 0;
    for(;;) {
        i = 0;
        do {
            if((c = xgetc(&in, inl)) < 0) break;
            if(c > 127) {
                c -= 127;
                while((c > 0) && (i < 256)) {
                    stack[i * 2] = i;
                    c--;
                    i++;
                }
            }
            c++;
            while((c > 0) && (i < 256)) {
                if((n = xgetc(&in, inl)) < 0) break;
                stack[i * 2] = n;
                if(i != n) {
                    if((n = xgetc(&in, inl)) < 0) break;
                    stack[(i * 2) + 1] = n;
                }
                c--;
                i++;
            }
        } while(i < 256);

        if((n = xgetc(&in, inl)) < 0) break;
        size = n;
        if((n = xgetc(&in, inl)) < 0) break;
        size |= (n << 8);

        while(size || count) {
            if(count) {
                count--;
                n = stack[count + 512];
            } else {
                if((n = xgetc(&in, inl)) < 0) break;
                size--;
            }
            c = stack[n * 2];
            if(n == c) {
                if(o >= outl) return(-1);
                *o++ = n;
            } else {
                if((count + 512 + 2) > sizeof(stack)) return(-1);
                stack[count + 512] = stack[(n * 2) + 1];
                stack[count + 512 + 1] = c;
                count += 2;
            }
        }
    }
    if(fill_outsz) {    // this is what is wanted by the format
        memset(o, 0, outl - o);
        o = outl;
    }
    return(o - out);
}

unsigned char * str_replace(
    unsigned char const * const original,
    unsigned char const * const pattern,
    unsigned char const * const replacement
) {
  size_t const replen = strlen(replacement);
  size_t const patlen = strlen(pattern);
  size_t const orilen = strlen(original);

  size_t patcnt = 0;
  const unsigned char * oriptr;
  const unsigned char * patloc;

  // find how many times the pattern occurs in the original string
  for (oriptr = original; patloc = strstr(oriptr, pattern); oriptr = patloc + patlen)
  {
    patcnt++;
  }

  {
    // allocate memory for the new string
    size_t const retlen = orilen + patcnt * (replen - patlen);
    unsigned char * const returned = (unsigned char *) malloc( sizeof(unsigned char) * (retlen + 1) );

    if (returned != NULL)
    {
      // copy the original string,
      // replacing all the instances of the pattern
      unsigned char * retptr = returned;
      for (oriptr = original; patloc = strstr(oriptr, pattern); oriptr = patloc + patlen)
      {
        size_t const skplen = patloc - oriptr;
        // copy the section until the occurence of the pattern
        strncpy(retptr, oriptr, skplen);
        retptr += skplen;
        // copy the replacement
        strncpy(retptr, replacement, replen);
        retptr += replen;
      }
      // copy the rest of the string.
      strcpy(retptr, oriptr);
    }
    return returned;
  }
}


int main(int argc, char *argv[])
{
    /* argv[1] = bpe_string */
    /* argv[2] = bpe_size
    /* argv[3] = unbpe_size */
    int const bpe_size   = atoi(argv[2]);
    int const unbpe_size = atoi(argv[3]);
    unsigned char *bpe_str[sizeof(argv[1])];
    *bpe_str = str_replace(argv[1], "\n\n\n", "\x00");
    *bpe_str = str_replace(*bpe_str, "\r\r\r", "\'");
    *bpe_str = str_replace(*bpe_str, "\t\t\t", "\"");
    *bpe_str = str_replace(*bpe_str, "\a\a\a", " ");
    unsigned char *unbpe_str[unbpe_size];
    yuke_bpe(*bpe_str,bpe_size,*unbpe_str,unbpe_size, 1);
    return 0;
}


Edited:Update the code,the program now works.But when I pass a valid BPE buffer,it crashes.The BPE buffer contains the compressed data,read from offset 0x10 to end of a BPE file.I have attached a bpe file as example.In the 7z archive,file000000.bpe is the correct bpe file,file000000.bpe.dat is the uncompressed file,file000000.re.bpe is the incorrect bpe file.Here is my parameter:
1- The BPE buffer,read from offset 0x10 of file000000.bpe
2- The BPE size,in this case is 0x11D1
3- The output buffer(a char* array of [parameter 4])
4- The uncompressed data size,in this case is 0x2C14

Here is the python script to open to call the C function.First compile the C program as 'unbpe_mem.exe' then drop it in the tools folder

Code: Select all

def decompress_mem(bpe_string,unbpe_size):
    bpe_size = len(bpe_string)
    bpe_string = bpe_string.replace('\x00', '\n\n\n')
    bpe_string = bpe_string.replace('\''  , '\r\r\r')
    bpe_string = bpe_string.replace('\"'  , '\t\t\t')
    bpe_string = bpe_string.replace(' '   , '\a\a\a')
    result = subprocess.call("tools\\unbpe_mem.exe \"%s\" %d %d" % (bpe_string,bpe_size,unbpe_size))
    return result

a = open('file000000.bpe', 'rb')
a.seek(0x10)
BPE_data = a.read()
a.close()
a = open('file000000.bpe.dat', 'rb')
unBPE_size = len(a.read())
decompress_mem(BPE_data, unBPE_size)


Edit: I got it working,thanks aluigi
eri619
Posts: 21
Joined: Sun Nov 29, 2015 11:06 pm

Re: Unbpe source code

Post by eri619 »

we now have a proper BPE Compression tool for the first time ever..
eatrawmeat391
Posts: 9
Joined: Sun Jul 17, 2016 5:23 am

Re: Unbpe source code

Post by eatrawmeat391 »

Actually,VicThrash36 made it first.I was very suprised to see that after I released the tool