wiiqt6/WiiQt/u8.cpp

#include "u8.h"
#include "lz77.h"
#include "tools.h"
//#include "md5.h"
#include "ash.h"

#define U8_HEADER_ALIGNMENT 0x20 //wiibrew says 0x40, but wii.cs is using 0x20 and i havent heard of any bricks because of it
static quint32 swap24( quint32 i )
{
#if Q_BYTE_ORDER == Q_BIG_ENDIAN
    return i & 0xffffff;
#else
    int r = qFromBigEndian( i );
    return r >> 8;
#endif
}

U8::U8( bool initialize, int type, const QStringList &names )
{
    ok = false;
    isLz77 = false;
    paths.clear();
    nestedU8s.clear();
    fst = NULL;
    fstSize = 0;
    headerType = type;
    imetNames = names;
    if( !initialize )
        return;

    //headerType = type;
    //imetNames = names;

    ok = CreateEmptyData();
}

bool U8::CreateEmptyData()
{
    isLz77 = false;
    nestedU8s.clear();
    data = QByteArray( 0x20, '\0' );

    //create a generic u8 with no real info in it.  only a fst with a root node
    QBuffer buf( &data );
    if( !buf.open( QBuffer::WriteOnly ) )
    {
        qWarning() << "Can't create buffer";
        data.clear();
        return false;
    }
    buf.putChar( 'U' );
    buf.putChar( '\xAA' );
    buf.putChar( '8' );
    buf.putChar( '\x2D' );

    rootnode_offset = 0x00000020;
    quint32 tmp = qFromBigEndian( rootnode_offset );
    buf.write( (const char*)&tmp, 4 );

    fstSize = 0x0000000D;
    tmp = qFromBigEndian( fstSize );
    buf.write( (const char*)&tmp, 4 );

    data_offset = qFromBigEndian( 0x00000040 );
    tmp = qFromBigEndian( data_offset );
    buf.write( (const char*)&tmp, 4 );
    buf.close();

    //create the new fst
    QByteArray fstArray = QByteArray( 0x20, '\0' );
    fst = (FEntry*)( fstArray.data() );
    fst[ 0 ].Type = 1;
    fst[ 0 ].FileLength = qFromBigEndian( 0x00000001 );
    NameOff = 0x0C;

    data.append( fstArray );

    //point the fst pointer to the new data
    fst = (FEntry*)( data.data() + rootnode_offset );


    ok = true;
    return true;
}

bool U8::RenameEntry( const QString &path, const QString &newName )
{
    qDebug() << "U8::RenameEntry(" << path << "," << newName << ")";
    if( !ok )
    {
        qWarning() << "U8::RenameEntry -> archive has no data";
        return false;
    }
    if( newName.contains( "/" ) )
    {
        qWarning() << "U8::RenameEntry -> newName cannot contain \'/\'";
        return false;
    }
    //check if this is a path to a file in a nested archive
    QMap<QString, U8 >::iterator i = nestedU8s.begin();
    while( i != nestedU8s.constEnd() )
    {
        if( path.startsWith( i.key() ) && path != i.key() )
        {
            QString subPath = path;
            subPath.remove( 0, i.key().size() + 1 );//remove the path of the archive itself + the slash

            bool ret = i.value().RenameEntry( subPath, newName );
            if( !ret )
            {
                qWarning() << "U8::RenameEntry -> error replacing data in child U8 archive";
                return false;
            }
            //qDebug() << "child entry updated, now replacing child archive in this one";
            return ReplaceEntry( i.key(),i.value().GetData() );
            //NOTE - after replacing the entry, "i" is no longer valid.  keep that in mind when changing this code.  its a bitch to track down this bug
        }
        i++;
    }
    //make sure the new filename doesnt already exist
    QString parentPath = path;
    while( parentPath.startsWith( "/") )//remove leading "/"s
        parentPath.remove( 0, 1 );
    while( parentPath.endsWith( "/") )//remove trailing "/"s
        parentPath.resize( parentPath.size() - 1 );

    int slash = parentPath.lastIndexOf( "/" );
    if( slash != -1 )
    {
        parentPath.chop( ( parentPath.size() - slash ) );
        parentPath += "/" + newName;
    }
    else
        parentPath = newName;

    if( FindEntry( parentPath ) != -1 )
    {
        qWarning() << "U8::RenameEntry ->" << parentPath << "already exists in the archive";
        return false;
    }

    //find the entry to rename
    int entryToRename = FindEntry( path );
    if( entryToRename == -1 )
    {
        qWarning() << "U8::RenameEntry" << path << "doesn\'t exists in the archive";
        qWarning() << "choices are" << paths;
        return false;
    }

    quint32 newNameLen = newName.size();
    quint32 oldNameLen = FstName( entryToRename ).size();
    int difference = newNameLen - oldNameLen;


    quint32 nFstSize = fstSize + difference;
    int dataAdjustment = 0;
    if( RU( nFstSize, U8_HEADER_ALIGNMENT ) < RU( fstSize, U8_HEADER_ALIGNMENT ) )
        dataAdjustment = - RU( ( fstSize - nFstSize ), U8_HEADER_ALIGNMENT );

    else if( RU( nFstSize, U8_HEADER_ALIGNMENT ) > RU( fstSize, U8_HEADER_ALIGNMENT ) )
        dataAdjustment = RU( ( nFstSize - fstSize ), U8_HEADER_ALIGNMENT );

    qDebug() << "old size:" << hex << oldNameLen\
            << "new size:" << hex << newNameLen\
            << "difference:" << hex << difference
            << "dataAdjustment:" << hex << dataAdjustment;
    QByteArray nFstData( ( qFromBigEndian( fst[ 0 ].FileLength ) ) * 0xc, '\0' );
    FEntry *nfst = (FEntry*)( nFstData.data() );
    //make the new root entry
    nfst[ 0 ] = fst[ 0 ];



    //the new data payload ( just copy the existing one )
    QByteArray nPayload = data.right( data.size() - data_offset );
    QByteArray nNameTable;
    nNameTable.append( '\0' );//add a null byte for the root name

    quint32 cnt = qFromBigEndian( fst[ 0 ].FileLength );//the number of entries


    for( quint32 i = 1; i < cnt; i++ )
    {
        FEntry *e = &fst[ i ];//old entry
        FEntry *ne = &nfst[ i ];//new entry

        ne->NameOffset = swap24( nNameTable.size() );//offset to the new entry's name in the string table

        //add the name to the new name table
        if( i == (quint32)entryToRename )
        {
            nNameTable.append( newName.toLatin1() );
        }
        else
            nNameTable.append( FstName( e ).toLatin1() );

        nNameTable.append( '\0' );
        if( e->Type )//nothing special to change for directories except the name offset
        {
            ne->Type = 1;
            ne->ParentOffset = e->ParentOffset;
            ne->NextOffset = e->NextOffset;
        }
        else//only need to change the name offset & the data offset for files
        {
            ne->Type = 0;
            ne->FileOffset = qFromBigEndian( qFromBigEndian( e->FileOffset ) + dataAdjustment );// + qFromBigEndian( dataAdjustment );
            //qFromBigEndian( (quint32)( 0x20 + RU( U8_HEADER_ALIGNMENT, nFstSize ) + nPayload.size() ) );
            ne->FileLength = e->FileLength;
            qDebug() << "old offset" << hex << qFromBigEndian( e->FileOffset ) << "new offset" << hex << qFromBigEndian( e->FileOffset ) + dataAdjustment;
        }

    }
    //now put all the parts together to make a new U8 archive
    nFstData.append( nNameTable );

    data = QByteArray( 0x20, '\0' );
    QBuffer buf( &data );
    if( !buf.open( QBuffer::WriteOnly ) )
    {
        qWarning() << "U8::AddEntry -> Can't create buffer";
        return -1;
    }
    buf.putChar( 'U' );
    buf.putChar( '\xAA' );
    buf.putChar( '8' );
    buf.putChar( '\x2D' );

    rootnode_offset = 0x00000020;
    quint32 t = qFromBigEndian( rootnode_offset );
    buf.write( (const char*)&t, 4 );

    fstSize = nFstData.size();
    t = qFromBigEndian( fstSize );
    buf.write( (const char*)&t, 4 );

    data_offset = RU( ( 0x20 + fstSize ), U8_HEADER_ALIGNMENT );
    t = qFromBigEndian( data_offset );
    buf.write( (const char*)&t, 4 );
    buf.close();

    data.append( nFstData );//add the fst
    int padding = data_offset - data.size();//pad to 'data_offset' after the fst
    if( padding )
    {
        data.append( QByteArray( padding, '\0' ) );
    }
    data.append( nPayload );//add the actual file data

    padding = RU( data.size(), 0x20 ) - data.size();//pad the entire thing to 0x20 bytes TOTO: should probably already be done, and this step is not really necessary
    if( padding )
    {
        data.append( QByteArray( padding, '\0' ) );
    }



    CreateEntryList();

    //hexdump( data, 0, 0x100 );
    //hexdump( data );
    return true;
}

bool U8::ReplaceEntry( const QString &path, const QByteArray &nba, bool autoCompress )
{
    //qDebug() << "U8::ReplaceEntry(" << path << ")";
    if( !ok )
    {
        qWarning() << "U8::ReplaceEntry -> archive has no data";
        return false;
    }
    //check if this is a path to a file in a nested archive
    QMap<QString, U8 >::iterator i = nestedU8s.begin();
    while( i != nestedU8s.constEnd() )
    {
        if( path.startsWith( i.key() ) && path != i.key() )
        {
            QString subPath = path;
            subPath.remove( 0, i.key().size() + 1 );//remove the path of the archive itself + the slash
            //qDebug() << "replacing file" << subPath << "in child U8" << i.key();

            U8 ch = i.value();
            QString chPath = i.key();
            //qDebug() << "replacing" << subPath << "in child archive";
            bool ret = ch.ReplaceEntry( subPath, nba, autoCompress );
            if( !ret )
            {
                qWarning() << "U8::ReplaceEntry -> error replacing data in child U8 archive";
                return false;
            }
            //qDebug() << "child entry updated, now replacing child archive in this one";
            return ReplaceEntry( chPath, ch.GetData() );
            //NOTE - after replacing the entry, "i" is no longer valid.  keep that in mind when changing this code.  its a bitch to track down this bug
        }
        i++;
    }
    //find the entry to replace
    int entryToReplace = FindEntry( path );
    if( entryToReplace == -1 )
    {
        qWarning() << "U8::ReplaceEntry" << path << "doesn\'t exists in the archive";
        qWarning() << "choices are" << paths;
        return false;
    }
    if( qFromBigEndian( fst[ entryToReplace ].Type ) )
    {
        qWarning() << "U8::ReplaceEntry -> replacing a directory is not supported" << path;
        return false;
    }


    /*qDebug() << "old size:" << hex << oldSizePadded\
        << "new size:" << hex << newSizePadded\
        << "difference:" << hex << difference;*/
    QByteArray newData = nba;
    if( autoCompress )
    {
        QByteArray oldData = data.mid( qFromBigEndian( fst[ entryToReplace ].FileOffset ), qFromBigEndian( fst[ entryToReplace ].FileLength ) );
        bool oldCompressed = ( LZ77::GetLz77Offset( oldData ) > -1 || IsAshCompressed( oldData ) );
        if( oldCompressed && LZ77::GetLz77Offset( newData ) == -1 && !IsAshCompressed( newData ) )
        {
            newData = LZ77::Compress( newData );
        }
    }

    quint32 newSizePadded = RU( newData.size(), 0x20 );
    quint32 oldSizePadded = RU( qFromBigEndian( fst[ entryToReplace ].FileLength ), 0x20 );
    int difference = newSizePadded - oldSizePadded;

    data.remove( qFromBigEndian( fst[ entryToReplace ].FileOffset ), oldSizePadded );
    QByteArray newPaddedData = newData;
    if( newSizePadded > (quint32)newPaddedData.size() )
    {
        newPaddedData.append( QByteArray( newSizePadded - newPaddedData.size(), '\0' ) );
    }
    data.insert( qFromBigEndian( fst[ entryToReplace ].FileOffset ), newPaddedData );

    if( newSizePadded == oldSizePadded )//the sized match, so nothing else to change
        return true;

    //point the fst to the new data
    fst = (FEntry*)( data.data() + rootnode_offset );

    //edit the changed size
    FEntry *e = &fst[ entryToReplace ];
    e->FileLength = qFromBigEndian( newData.size() );
    //hexdump( (const void*)e, sizeof( FEntry ) );

    //edit all the file offsets after the changed item
    quint32 cnt = qFromBigEndian( fst[ 0 ].FileLength );
    for( quint32 i = entryToReplace + 1; i < cnt; i++ )
    {
        FEntry *e = &fst[ i ];//old entry
        if( e->Type )//nothing changes for directories
            continue;

        //qDebug() << "changed" << FstName( e ) << "offset from" << hex << qFromBigEndian( fst[ i ].FileOffset ) << "to" << qFromBigEndian( fst[ i ].FileOffset ) + difference;
        e->FileOffset = qFromBigEndian( qFromBigEndian( fst[ i ].FileOffset ) + difference );
    }
    CreateEntryList();

    //hexdump( data, 0, 0x100 );
    //hexdump( data );
    return true;

}

bool U8::RemoveEntry( const QString &path )
{
    //qDebug() << "U8::RemoveEntry(" << path << ")";
    if( !ok )
    {
        qWarning() << "U8::RemoveEntry -> archive has no data";
        return false;
    }
    //check if this is a path to a file in a nested archive
    QMap<QString, U8 >::iterator i = nestedU8s.begin();
    while( i != nestedU8s.constEnd() )
    {
        if( path.startsWith( i.key() ) && path != i.key() )
        {
            QString subPath = path;
            subPath.remove( 0, i.key().size() + 1 );//remove the path of the archive itself + the slash

            U8 ch = i.value();
            QString chPath = i.key();
            //qDebug() << "deleting" << subPath << "in child archive";

            bool ret = ch.RemoveEntry( subPath );
            if( !ret )
            {
                qWarning() << "U8::RemoveEntry -> error replacing data in child U8 archive";
                return false;
            }
            return ReplaceEntry( chPath, ch.GetData() );//insert the new nested archive in this one
            //NOTE - after replacing the entry, "i" is no longer valid.  keep that in mind when changing this code.  its a bitch to track down this bug
        }
        i++;
    }
    //find the entry to delete
    int entryToDelete = FindEntry( path );
    if( entryToDelete == -1 )
    {
        qWarning() << "U8::RemoveEntry" << path << "doesn\'t exists in the archive";
        return false;
    }

    //create a list of all the directory indexes that my have their size changed
    QList<int> parents;
    int parent;
    quint32 numDeletedEntries;
    quint32 fstLenToDelete;
    if( fst[ entryToDelete ].Type )//deleting a directory
    {
        parent = qFromBigEndian( fst[ entryToDelete ].ParentOffset );
        while( parent )
        {
            parents << parent;
            parent = qFromBigEndian( fst[ parent ].ParentOffset );
        }
        numDeletedEntries = qFromBigEndian( fst[ entryToDelete ].NextOffset ) - entryToDelete;
        fstLenToDelete = 0;
        for( quint32 i = entryToDelete; i < qFromBigEndian( fst[ entryToDelete ].NextOffset ); i ++ )
        {
            fstLenToDelete += FstName( i ).size() + 1 + 0xc;
        }

    }
    else//deleting a file
    {
        parent = entryToDelete - 1;
        while( parent )
        {
            if( fst[ parent ].Type && qFromBigEndian( fst[ parent ].NextOffset ) > (quint32)entryToDelete )
            {
                parents << parent;
                parent = qFromBigEndian( fst[ parent ].ParentOffset );
            }
            else
                parent--;
        }
        numDeletedEntries = 1;
        fstLenToDelete = FstName( entryToDelete ).size() + 1 + 0xc;

    }


    //qDebug() << "will delete" << numDeletedEntries << "entries";

    quint32 nFstSize = fstSize - fstLenToDelete;//old fst size - 0xc bytes for each deleted entry and the length of its name + 1 null byte
    //qDebug() << "nFstSize" << nFstSize;
    QByteArray nFstData( ( qFromBigEndian( fst[ 0 ].FileLength ) - numDeletedEntries ) * 0xc, '\0' );
    FEntry *nfst = (FEntry*)( nFstData.data() );

    //the new data payload ( all the files )
    QByteArray nPayload;
    QByteArray nNameTable;
    nNameTable.append( '\0' );//add a null byte for the root name

    quint32 cnt = qFromBigEndian( fst[ 0 ].FileLength );//the number of entries + 1 for the new one

    //make the new root entry
    nfst[ 0 ] = fst[ 0 ];
    nfst[ 0 ].FileLength = qFromBigEndian( cnt - numDeletedEntries );

    QList<int> movedDirs;//keep a list of the directories that are shifted back so their children can be shifted too
    for( quint32 i = 1; i < cnt; i++ )
    {
        if( i >= (quint32)entryToDelete && i < (quint32)( entryToDelete + numDeletedEntries ) )//skip deleted entries
            continue;

        //copy old entries and adjust as needed
        quint8 adj = i < (quint32)entryToDelete ? 0 : numDeletedEntries;
        quint32 ni = i - adj;

        //qDebug() << "keeping" << FstName( i ) << "in the new archive ( moved from" << hex << i << "to" << hex << ni << ")";
        //if( parents.contains( i ) )
        //qDebug() << "\tthis is a parent of the deleted item";

        FEntry *e = &fst[ i ];//old entry
        FEntry *ne = &nfst[ ni ];//new entry

        nfst[ ni ].NameOffset = swap24( nNameTable.size() );//offset to the new entry's name in the string table

        //add the name to the new name table
        nNameTable.append( FstName( e ).toLatin1() );
        nNameTable.append( '\0' );


        if( e->Type )//directory
        {
            ne->Type = 1;
            if( !adj )//directories before the new entry
            {
                ne->ParentOffset = e->ParentOffset;
                ne->NextOffset = qFromBigEndian( qFromBigEndian( e->NextOffset ) \
                                                 - ( parents.contains( i ) ? numDeletedEntries : 0 ) );
            }
            else//directories after the new entry
            {
                ne->ParentOffset = qFromBigEndian( qFromBigEndian( e->ParentOffset ) \
                                                   - ( movedDirs.contains( qFromBigEndian( e->ParentOffset ) ) ? numDeletedEntries : 0 ) );
                ne->NextOffset = qFromBigEndian( qFromBigEndian( e->NextOffset ) - numDeletedEntries );

                movedDirs << i;

                //qDebug() << "e.parent:" << hex << qFromBigEndian( e->ParentOffset ) << "movedDirs:" << movedDirs;
                //hexdump( (const void*)ne, sizeof( FEntry) );
            }
        }
        else//file
        {
            ne->Type = 0;
            ne->FileOffset = \
                             qFromBigEndian( (quint32)( 0x20 + RU( nFstSize, U8_HEADER_ALIGNMENT ) + nPayload.size() ) );
            ne->FileLength = e->FileLength;
            nPayload.append( data.mid( qFromBigEndian( e->FileOffset ), qFromBigEndian( e->FileLength ) ) );
            int padding = RU( nPayload.size(), 0x20 ) - nPayload.size();//pad to 0x20 bytes between files
            if( padding )
            {
                nPayload.append( QByteArray( padding, '\0' ) );
            }
            //qDebug() << "writing fileOffset of" << hex << ni << hex << (quint32)( 0x20 + RU( U8_HEADER_ALIGNMENT, nFstSize ) + nPayload.size() );
        }
        //hexdump( (const void*)ne, sizeof( FEntry) );

    }
    //hexdump( nFstData );

    //now put all the parts together to make a new U8 archive
    nFstData.append( nNameTable );

    data = QByteArray( 0x20, '\0' );
    QBuffer buf( &data );
    if( !buf.open( QBuffer::WriteOnly ) )
    {
        qWarning() << "U8::AddEntry -> Can't create buffer";
        return -1;
    }
    buf.putChar( 'U' );
    buf.putChar( '\xAA' );
    buf.putChar( '8' );
    buf.putChar( '\x2D' );

    rootnode_offset = 0x00000020;
    quint32 t = qFromBigEndian( rootnode_offset );
    buf.write( (const char*)&t, 4 );

    fstSize = nFstData.size();
    t = qFromBigEndian( fstSize );
    buf.write( (const char*)&t, 4 );

    data_offset = RU( ( 0x20 + fstSize ), U8_HEADER_ALIGNMENT );
    t = qFromBigEndian( data_offset );
    buf.write( (const char*)&t, 4 );
    buf.close();

    data.append( nFstData );//add the fst
    int padding = data_offset - data.size();//pad to 'data_offset' after the fst
    if( padding )
    {
        data.append( QByteArray( padding, '\0' ) );
    }
    data.append( nPayload );//add the actual file data

    padding = RU( data.size(), 0x20 ) - data.size();//pad the entire thing to 0x20 bytes TOTO: should probably already be done, and this step is not really necessary
    if( padding )
    {
        data.append( QByteArray( padding, '\0' ) );
    }

    //make sure the pirvate variables are correct
    fst = (FEntry*)( data.data() + rootnode_offset );
    NameOff = 0x0C * qFromBigEndian( fst[ 0 ].FileLength );

    CreateEntryList();

    //hexdump( data );
    //qDebug() << "dataSize after removal:" << hex << data.size();

    return true;

}

int U8::AddEntry( const QString &path, int type, const QByteArray &newData )
{
    //qDebug() << "U8::AddEntry(" << path << "," << type << "," << hex << newData.size() << ")";
    //make sure there is actually data to manipulate
    if( !ok && !CreateEmptyData() )
    {
        return -1;
    }
    //check if this is a path to a file in a nested archive
    QMap<QString, U8 >::iterator i = nestedU8s.begin();
    while( i != nestedU8s.constEnd() )
    {
        if( path.startsWith( i.key() ) && path != i.key() )
        {
            QString subPath = path;
            //make a copy of the string, as "i" will become invalid on replace entry, and create a crash that is a bitch to track down
            QString thisPath = i.key();
            U8 chVal = i.value();
            //qDebug() << "adding entry" << subPath << "in child U8" << thisPath;
            subPath.remove( 0, thisPath.size() + 1 );//remove the path of the archive itself + the slash
            bool ret = chVal.AddEntry( subPath, type, newData );
            if( !ret )
            {
                qWarning() << "U8::AddEntry -> error replacing data in child U8 archive";
                return false;
            }

            if( !ReplaceEntry( thisPath, chVal.GetData() ) )//insert the new nested archive in this one
                return -1;

            //qDebug() << "done replacing the child entry in this archive, finding the index to return (" << thisPath << ")";
            return FindEntry( thisPath );//just return the index of the nested archive
        }
        i++;
    }

    //make sure this path doesnt already exist
    if( FindEntry( path ) != -1 )
    {
        qWarning() << "U8::AddEntry" << path << "already exists in the archive";
        return -1;
    }
    //find the parent for this new entry
    QString parentPath = path;
    while( parentPath.startsWith( "/") )//remove leading "/"s
        parentPath.remove( 0, 1 );
    while( parentPath.endsWith( "/") )//remove trailing "/"s
        parentPath.resize( parentPath.size() - 1 );

    int parent = 0;
    int slash = parentPath.lastIndexOf( "/" );
    if( slash != -1 )
    {
        //parentPath.resize( ( parentPath.size() - slash ) - 2 );
        parentPath.chop( ( parentPath.size() - slash ) );
        parent = FindEntry( parentPath );
        if( parent == -1 )//need to create subfolders for this fucker
        {
            parent = AddEntry( parentPath, 1 );
            if( parent == -1 )
            {
                qWarning() << "U8::AddEntry -> could not create subfolders for" << path;
                return -1;
            }
        }
        if( !fst[ parent ].Type )//parent maps to a file, not folder
        {
            qWarning() << "U8::AddEntry -> can't add child entry to a file";
            return -1;
        }
    }
    //qDebug() << "adding entry to" << FstName( parent );

    //find the index to insert the new path
    int newEntry = qFromBigEndian( fst[ parent ].NextOffset );

    //create a list of all the directory indexes that my have their size changed to hold the new entry
    QList<int> parents;
    while( parentPath.contains( "/" ) )
    {
        parents << FindEntry( parentPath );
        int slash = parentPath.lastIndexOf( "/" );
        parentPath.chop( ( parentPath.size() - slash ) );
    }
    parents << FindEntry( parentPath );
    //qDebug() << "parents" << parents;

    //create the new data

    //the new fst
    QString name = path;//get the actual name of this entry
    int sl = name.lastIndexOf( "/" );
    if( sl != -1 )
    {
        name.remove( 0, sl + 1 );
    }

    quint32 nFstSize = fstSize + 0x0C + name.size() + 1;//old fst size + 0xc bytes for the new entry + room for the new name in the string table
    QByteArray nFstData( ( qFromBigEndian( fst[ 0 ].FileLength ) + 1 ) * 0xc, '\0' );
    FEntry *nfst = (FEntry*)( nFstData.data() );

    //the new data payload ( all the files )
    QByteArray nPayload;
    QByteArray nNameTable;
    nNameTable.append( '\0' );//add a null byte for the root name

    quint32 cnt = qFromBigEndian( fst[ 0 ].FileLength ) + 1;//the number of entries + 1 for the new one

    //make the new root entry
    nfst[ 0 ] = fst[ 0 ];
    nfst[ 0 ].FileLength = qFromBigEndian( cnt );

    QList<int> movedDirs;//keep a list of the directories that are shifted back so their children can be shifted too
    for( quint32 i = 1; i < cnt; i++ )
    {
        nfst[ i ].NameOffset = swap24( nNameTable.size() );//offset to the new entry's name in the string table
        if( i == (quint32)newEntry )//add the new entry
        {
            FEntry *ne = &nfst[ i ];
            if( type )
            {
                ne->Type = 1;
                ne->ParentOffset = qFromBigEndian( parent );
                ne->NextOffset = qFromBigEndian( i + 1 );
            }
            else
            {
                ne->Type = 0;
                ne->FileLength = qFromBigEndian( (quint32)newData.size() );
                ne->FileOffset =\
                                qFromBigEndian( (quint32)( 0x20 + RU( nFstSize, U8_HEADER_ALIGNMENT ) + nPayload.size() ) );
                nPayload.append( newData );
                int padding = RU( nPayload.size(), 0x20 ) - nPayload.size();//pad to 0x20 bytes between files
                if( padding )
                {
                    nPayload.append( QByteArray( padding, '\0' ) );
                }
            }
            nNameTable.append( name.toLatin1() );//add this entry's name to the table
            nNameTable.append( '\0' );
            continue;
        }

        //copy old entries and adjust as needed
        quint8 adj = i < (quint32)newEntry ? 0 : 1;
        quint32 ni = i - adj;

        FEntry *e = &fst[ ni ];//old entry
        FEntry *ne = &nfst[ i ];//new entry

        //add the name to the new name table
        nNameTable.append( FstName( e ).toLatin1() );
        nNameTable.append( '\0' );
        if( e->Type )
        {
            ne->Type = 1;
            if( !adj )//directories before the new entry
            {
                ne->ParentOffset = e->ParentOffset;
                ne->NextOffset = qFromBigEndian( qFromBigEndian( e->NextOffset ) \
                                                 + ( parents.contains( i ) ? 1 : 0 ) );
            }
            else//directories after the new entry
            {
                movedDirs << ni;
                ne->ParentOffset = qFromBigEndian( qFromBigEndian( e->ParentOffset ) \
                                                   + ( movedDirs.contains( qFromBigEndian( e->ParentOffset ) ) ? 1 : 0 ) );
                ne->NextOffset = qFromBigEndian( qFromBigEndian( e->NextOffset ) + 1 );
            }
        }
        else//all file entries
        {
            ne->Type = 0;
            ne->FileOffset = \
                             qFromBigEndian( (quint32)( 0x20 + RU( nFstSize, U8_HEADER_ALIGNMENT ) + nPayload.size() ) );
            ne->FileLength = e->FileLength;
            nPayload.append( data.mid( qFromBigEndian( e->FileOffset ), qFromBigEndian( e->FileLength ) ) );
            int padding = RU( nPayload.size(), 0x20 ) - nPayload.size();//pad to 0x20 bytes between files
            if( padding )
            {
                nPayload.append( QByteArray( padding, '\0' ) );
            }
        }

    }
    //now put all the parts together to make a new U8 archive
    nFstData.append( nNameTable );

    data = QByteArray( 0x20, '\0' );
    QBuffer buf( &data );
    if( !buf.open( QBuffer::WriteOnly ) )
    {
        qWarning() << "U8::AddEntry -> Can't create buffer";
        return -1;
    }
    buf.putChar( 'U' );
    buf.putChar( '\xAA' );
    buf.putChar( '8' );
    buf.putChar( '\x2D' );

    rootnode_offset = 0x00000020;
    quint32 t = qFromBigEndian( rootnode_offset );
    buf.write( (const char*)&t, 4 );

    fstSize = nFstData.size();
    t = qFromBigEndian( fstSize );
    buf.write( (const char*)&t, 4 );

    data_offset = RU( ( 0x20 + fstSize ), U8_HEADER_ALIGNMENT );
    t = qFromBigEndian( data_offset );
    buf.write( (const char*)&t, 4 );
    buf.close();

    data.append( nFstData );//add the fst
    int padding = data_offset - data.size();//pad to 'data_offset' after the fst
    if( padding )
    {
        data.append( QByteArray( padding, '\0' ) );
    }
    data.append( nPayload );//add the actual file data

    padding = RU( data.size(), 0x20 ) - data.size();//pad the entire thing to 0x20 bytes TOTO: should probably already be done, and this step is not really necessary
    if( padding )
    {
        data.append( QByteArray( padding, '\0' ) );
    }

    //make sure the pirvate variables are correct
    fst = (FEntry*)( data.data() + rootnode_offset );
    NameOff = 0x0C * qFromBigEndian( fst[ 0 ].FileLength );
    //qDebug() << "done adding the entry";

    CreateEntryList();

    //hexdump( data );
    return newEntry;
}

void U8::CreateEntryList()
{
    //qDebug() << "U8::CreateEntryList";
    paths.clear();
    nestedU8s.clear();
    fst = (FEntry*)( data.data() + rootnode_offset );
    quint32 cnt = qFromBigEndian( fst[ 0 ].FileLength );
    if( ( cnt * 0x0d ) + rootnode_offset > (quint32)data.size() )
    {
        qWarning() << "U8::CreateEntryList -> invalid archive";
        ok = false;
        data.clear();
        paths.clear();
        fst = NULL;
        return;
    }
    NameOff = cnt * 0x0C;
    bool fixWarn = false;//ony print the warning 1 time

    //qDebug() << "cnt" << hex << cnt;
    for( quint32 i = 1; i < cnt; ++i )//this is not the most effecient way to do things, but it seems to work ok and these archives are small enough that it happens fast anyways
    {
        //start at the beginning of the fst and enter every directory whos "nextoffset" is greater than this index,
        //adding the names along the way and hopefully finding "i"
        QString path;
        quint32 current = 1;
        quint32 folder = 0;
        while( 1 )
        {
            if( !current )
            {
                qWarning() << "U8::CreateEntryList -> error parsing the archive";
                break;
            }
            if( current == i )
            {
                path += FstName( i );
                if( fst[ current ].Type )
                {
                    //sanity check:  make sure the parent offset is correct
                    if( folder != qFromBigEndian( fst[ current ].ParentOffset ) )
                    {
                        qWarning() << "U8::CreateEntryList -> error parsing the archive - recursion mismatch in"
                                << path << "expected:" << hex << folder << "got:" << hex << qFromBigEndian( fst[ current ].ParentOffset )\
                                << "(" << FstName( qFromBigEndian( fst[ current ].ParentOffset ) ) << ")";

                        //some tools use "recursion" instead of "parent offset".
                        //it just coincidentally works for archives such as the opening.bnr and .app since they dont have many folders to mess up
                        if( qFromBigEndian( fst[ current ].ParentOffset ) == (quint32)path.count( "/" ) )
                        {
                            if( !fixWarn )
                            {
                                qWarning() << "This archive was made by a broken tool such as U8Mii.  I'm trying to fix it, but I can't make any promises";
                                fixWarn = true;
                            }
                            fst[ current ].ParentOffset = qFromBigEndian( folder );
                        }
                    }
                    path += "/";
                    paths << path;
                }
                else
                {
                    //add this file to the list of entries
                    paths << path;

                    //check if this file is really a U8 archive, and add it to the list of children
                    QByteArray chData = data.mid( qFromBigEndian( fst[ current ].FileOffset ), qFromBigEndian( fst[ current ].FileLength ) );
                    if( IsU8( chData ) )
                    {
                        U8 child( chData );
                        if( child.IsOK() )
                        {
                            nestedU8s.insert( path, child );
                            foreach( QString chPath, child.Entries() )
                            {
                                QString newPath = path + "/" + chPath;
                                paths << newPath;
                            }
                        }
                    }
                }
                break;
            }
            if( fst[ current ].Type && i < qFromBigEndian( fst[ current ].NextOffset ) )
            {
                path += FstName( current ) + "/";
                folder = current;
                current++;
                continue;

            }
            current = NextEntryInFolder( current, folder );
        }
        //paths << path;
    }
    //qDebug() << "done";
    //qDebug() << "paths" << paths;
}

U8::U8( const QByteArray &ba )
{
    //qDebug() << "U8::U8 dataSize:" << hex << ba.size();
    Load( ba );
}

void U8::Load( const QByteArray &ba )
{
    ok = false;
    isLz77 = false;
    headerType = U8_Hdr_none;
    paths.clear();
    imetNames.clear();
    /*if( ba.size() < 0x80 )
    {
    //qWarning() << "U8::Load:" << hex << ba.size();
    //qWarning() << "U8::Load -> where is the rest of the data?";
    return;
    }*/
    data = ba;
    if( IsAshCompressed( data ) )//decrypt ASH0 files
        data = DecryptAsh( data );

    quint32 tmp;
    int off = LZ77::GetLz77Offset( data );
    int off2 = GetU8Offset( data );
    if( off != -1 && ( off2 == -1 || ( off2 != -1 && off < off2 ) ) )
    {
        isLz77 = true;
        data = LZ77::Decompress( data );
        off2 = GetU8Offset( data );
    }

    if( off2 == -1 )
    {
        qDebug() << "This isnt a U8";
        return;
    }

    data = data.mid( off2 );

    QBuffer buf( &data );
    if( !buf.open( QBuffer::ReadOnly ) )
    {
        qWarning() << "Can't create buffer";
        return;
    }

    buf.seek( 4 );
    buf.read( ( char*)&tmp, 4 );
    rootnode_offset = qFromBigEndian( tmp );
    if( rootnode_offset != 0x20 )
    {
        qWarning() << "rootnodeOffset" << hex << rootnode_offset;
        qWarning() << hex << data.size();
        hexdump( data );
    }

    buf.read( ( char*)&tmp, 4 );
    fstSize = qFromBigEndian( tmp );

    buf.read( ( char*)&tmp, 4 );
    data_offset = qFromBigEndian( tmp );

    buf.close();

    CreateEntryList();


    ReadHeader( ba );
    ok = true;
    //qDebug() << "loaded" << paths;
}

bool U8::IsOK()
{
    return ok;
}

QString U8::FstName( const FEntry *entry )
{
    //qDebug() << "U8::FstName";
    if( entry == &fst[ 0 ] )
        return QString();

    int nameStart = swap24( entry->NameOffset ) + NameOff + rootnode_offset;
    return data.mid( nameStart, nameStart + 0x100 );
}

QString U8::FstName( quint32 i )
{
    if( i > fst[ 0 ].FileLength )
    {
        qWarning() << "U8::FstName -> index is out of range";
        return QString();
    }
    return FstName( &fst[ i ] );
}

quint32 U8::NextEntryInFolder( quint32 current, quint32 directory )
{
    //qDebug() << "U8::NextEntryInFolder(" << hex << current << "," << hex << directory << ")";
    quint32 next = ( fst[ current ].Type ? qFromBigEndian( fst[ current ].FileLength ) : current + 1 );
    //qDebug() << "next" << next << "len" << hex << qFromBigEndian( fst[ directory ].FileLength );
    if( next < qFromBigEndian( fst[ directory ].FileLength ) )
        return next;

    return 0;
}

int U8::FindEntry( const QString &str, int d )
{
    //qDebug() << "U8::FindEntry(" << str << "," << d << ")";
    if( str.isEmpty() )
        return 0;

    if( str.startsWith( "/" ) )
    {
        QString r = str;
        r.remove( 0, 1 );
        return FindEntry( r, d );
    }

    if( !fst[ d ].Type )
    {
        qDebug() << "ERROR!!" << FstName( &fst[ d ] ) << "is not a directory";
        return -1;
    }

    int next = d + 1;
    FEntry *entry = &fst[ next ];

    while( next )
    {
        QString item;
        int sl = str.indexOf( "/" );
        if( sl > -1 )
            item = str.left( sl );
        else
            item = str;

        if( FstName( entry ) == item )
        {
            if( item == str || item + "/" == str )//this is the item we are looking for
            {
                return next;
            }
            //this item is a parent folder of one we are looking for
            return FindEntry( str.right( ( str.size() - sl ) - 1 ), next );
        }
        //find the next entry in this folder
        next = NextEntryInFolder( next, d );
        entry = &fst[ next ];
    }

    //no entry with the given path was found
    return -1;
}

const QByteArray U8::GetData( const QString &str, bool onlyPayload )
{
    //qDebug() << "U8::GetData(" << str << ")";
    if( str.isEmpty() )//give the data for this whole archive
    {
        if( onlyPayload )
            return data;

        //qDebug() << "U8::GetData -> returning all the data for this archive.  headerType:" << headerType;

        QByteArray ret = data;

        switch( headerType )
        {
        case U8_Hdr_IMET_app:
        case U8_Hdr_IMET_bnr:
            ret = AddIMET( headerType );//data is lz77 compressed in this function if it needs to be
            break;
        case U8_Hdr_IMD5:
            {
                if( isLz77 )
                    ret = LZ77::Compress( ret );

                ret = AddIMD5( ret );

                //hexdump( ret, 0, 0x40 );
            }
            break;
        default:
            break;
        }
        return ret;
    }
    //check if this is a path to a file in a nested archive
    QMap<QString, U8 >::iterator i = nestedU8s.begin();
    while( i != nestedU8s.constEnd() )
    {
        if( str.startsWith( i.key() ) && str != i.key() )
        {
            QString subPath = str;
            subPath.remove( 0, i.key().size() + 1 );//remove the path of the archive itself + the slash
            return i.value().GetData( subPath, onlyPayload );
        }
        i++;
    }
    int index = FindEntry( str );
    if( index < 0 )
    {
        qWarning() << "U8::GetData" << str << "was not found in the archive";
        return QByteArray();
    }
    if( fst[ index ].Type )
    {
        qWarning() << "U8::GetData" << str << "is a directory";
        return QByteArray();
    }
    QByteArray ret = data.mid( qFromBigEndian( fst[ index ].FileOffset ), qFromBigEndian( fst[ index ].FileLength ) );
    //hexdump( ret, 0, 0x40 );
    if( onlyPayload )
    {
        if( LZ77::GetLz77Offset( ret ) != -1 )
            ret = LZ77::Decompress( ret );

        else if( IsAshCompressed( ret ) )
            ret = DecryptAsh( ret );
    }
    return ret;
}

quint32 U8::GetSize( const QString &str )
{
    if( str.isEmpty() )
    {
        return data.size();
    }
    int index = FindEntry( str );
    if( index < 0 )
    {
        qWarning() << "U8::GetSize" << str << "was not found in the archive";
        return 0;
    }
    if( fst[ index ].Type )
    {
        qWarning() << "U8::GetSize" << str << "is a directory";
        return 0;
    }
    return  qFromBigEndian( fst[ index ].FileLength );
}

const QStringList U8::Entries()
{
    return paths;
}

int U8::GetU8Offset( const QByteArray &ba )
{
    QByteArray start = ba.left( 5000 );
    return start.indexOf( "U\xAA\x38\x2d" );
}

bool U8::IsU8( const QByteArray &ba )
{
    QByteArray data = ba;
    if( IsAshCompressed( data ) )//decrypt ASH0 files
        data = DecryptAsh( data );

    int off = LZ77::GetLz77Offset( data );//decrypt LZ77
    if( off != -1 )
        data = LZ77::Decompress( data );

    QByteArray start = data.left( 5000 );
    return start.indexOf( "U\xAA\x38\x2d" ) != -1;
}

#define IMET_MAX_NAME_LEN 0x2a
typedef struct
{
    quint32 sig; // "IMET"
    quint32 unk1;
    quint32 unk2;
    quint32 filesizes[ 3 ];
    quint32 unk3;
    quint16 names[ 10 ][ IMET_MAX_NAME_LEN ];
    quint8 zeroes2[ 0x24c ];
    quint8 md5[ 0x10 ];
} IMET;

void U8::ReadHeader( const QByteArray &ba )
{
    //qDebug() << "U8::ReadHeader(" << hex << ba.size() << ")";
    //hexdump( ba );
    headerType = U8_Hdr_none;
    imetNames.clear();
    if( ba.startsWith( "IMD5" ) )//dont bother to read any more data
    {
        //qDebug() << "IMD5 header";
        headerType = U8_Hdr_IMD5;
        return;
    }

    QByteArray start = ba.left( sizeof( IMET ) + 0x80 );
    QBuffer buf( &start );
    if( !buf.open( QBuffer::ReadOnly ) )
    {
        qWarning() << "U8::ReadHeader Can't create buffer";
        return;
    }
    int off = start.indexOf( "IMET" );
    //qDebug() << "imet offset" << hex << off << "u8 offset" << hex << GetU8Offset( ba );
    if( off == 0x40 || off == 0x80 )//read imet header
    {
        if( off > GetU8Offset( ba ) )//in case somebody wants to put a IMET archive inside another U8 for whatever reason
            return;

        if( off == 0x40 )
            headerType = U8_Hdr_IMET_bnr;
        else
            headerType = U8_Hdr_IMET_app;

        buf.seek( off );
        IMET imet;
        buf.read( (char*)&imet, sizeof( IMET ) );
        buf.close();

        for( int h = 0; h < 10; h ++ )
        {
            QString name;
            for( int i = 0; i < IMET_MAX_NAME_LEN && imet.names[ h ][ i ] != 0; i++ )//no need to switch endian, 0x0000 is a palendrome
                name += QChar( qFromBigEndian( imet.names[ h ][ i ] ) );

            imetNames << name;
        }
        //done
        return;
    }
}
QByteArray U8::AddIMD5( QByteArray ba )
{
    quint32 size = ba.size();

    /*MD5 hash;
    hash.update( ba.data(), size );
    hash.finalize();*/

    size = qFromBigEndian( size );

    QByteArray imd5;
    QBuffer buf( &imd5 );
    if( !buf.open( QBuffer::WriteOnly ) )
    {
        qWarning() << "U8::AddIMD5 Can't create buffer";
        return QByteArray();
    }
    buf.putChar( 'I' );
    buf.putChar( 'M' );
    buf.putChar( 'D' );
    buf.putChar( '5' );

    buf.write( (const char*)&size, 4 );
    buf.seek( 0x10 );

    buf.write( GetMd5( ba ) );
    //buf.write( (const char*)hash.hexdigestChar(), 16 );
    buf.close();
    //qDebug() << hash.hexdigest().c_str();
    //hexdump( (void*)imd5.data(), 0x20 );

    ba.prepend( imd5 );

    return ba;
}

const QByteArray U8::AddIMD5()
{
    QByteArray ba = data;
    ba = AddIMD5( ba );
    return ba;
}

QByteArray U8::GetIMET( const QStringList &names, int paddingType, quint32 iconSize, quint32 bannerSize, quint32 soundSize )
{
    QByteArray ret = QByteArray( sizeof( IMET ) + 0x40, '\0' );
    QBuffer buf( &ret );
    if( !buf.open( QBuffer::WriteOnly ) )
    {
        qWarning() << "U8::GetIMET Can't create buffer";
        return QByteArray();
    }
    buf.seek( 0 + 0x40 );
    buf.putChar( 'I' );
    buf.putChar( 'M' );
    buf.putChar( 'E' );
    buf.putChar( 'T' );
    buf.seek( 6 + 0x40 );
    buf.putChar( '\x6' );
    buf.seek( 11 + 0x40 );
    buf.putChar( '\x3' );

    quint32 tmp = qFromBigEndian( iconSize );
    buf.write( (char*)&tmp, 4 );
    tmp = qFromBigEndian( bannerSize );
    buf.write( (char*)&tmp, 4 );
    tmp = qFromBigEndian( soundSize );
    buf.write( (char*)&tmp, 4 );

    int nameOffset = 0x1c + 0x40;
    int numNames = names.size();
    for( int i = 0; i < numNames; i++ )
    {
        QString name = names.at( i );
        int nameLen = name.size();
        buf.seek( nameOffset + ( IMET_MAX_NAME_LEN * i * 2 ) );
        for( int j = 0; j < nameLen; j++ )
        {
            quint16 letter = qFromBigEndian( name.at( j ).unicode() );
            buf.write( ( const char*)&letter, 2 );
        }
    }

    /*MD5 hash;
    hash.update( ret.data(), sizeof( IMET ) + 0x40 );
    hash.finalize();*/

    buf.seek( 0x5f0 );
    buf.write( GetMd5( ret ) );
    //buf.write( (const char*)hash.hexdigestChar(), 16 );
    buf.close();

    switch( paddingType )
    {
    case U8_Hdr_IMET_app:
        ret.prepend( QByteArray( 0x40, '\0' ) );
        break;
    case U8_Hdr_IMET_bnr:
        break;
    default:
        ret.remove( 0, 0x40 );
        break;
    }
    return ret;
}

const QStringList U8::IMETNames()
{
    return imetNames;
}

void U8::SetImetNames( const QStringList &names )
{
    imetNames.clear();
    imetNames = names;
}

const QByteArray U8::AddIMET( int paddingType )
{
    QByteArray ret = GetData( "meta/icon.bin" );

    quint32 iconSize;
    quint32 bannerSize;
    quint32 soundSize;

    //if these are lz77 compressed, write the uncompressed size to the header  ?bannerbomb?
    //otherwise write the size - 0x20 ( imd5 header size )

    if( LZ77::GetLz77Offset( ret ) != -1 )
        iconSize = LZ77::GetDecompressedSize( ret );
    else
        iconSize = ret.size() - 0x20;

    ret = GetData( "meta/banner.bin" );
    if( LZ77::GetLz77Offset( ret ) != -1 )
        bannerSize = LZ77::GetDecompressedSize( ret );
    else
        bannerSize = ret.size() - 0x20;

    ret = GetData( "meta/sound.bin" );
    if( LZ77::GetLz77Offset( ret ) != -1 )
        soundSize = LZ77::GetDecompressedSize( ret );
    else
        soundSize = ret.size() - 0x20;

    ret = GetIMET( imetNames, paddingType, iconSize, bannerSize, soundSize );
    if( isLz77 )//really?  can the entire banner be lz77 compressed?
        ret += LZ77::Compress( data );
    else
        ret += data;

    return ret;
}

int U8::GetHeaderType()
{
    return headerType;
}