wiiqt/WiiQt/u8.cpp

#include "u8.h"
#include "lz77.h"
#include "tools.h"
//#include "md5.h"
#include "ash.h"

#define U8_HEADER_ALIGNMENT 0x20 //wiibrew says 0x40, but wii.cs is using 0x20 and i havent heard of any bricks because of it
static quint32 swap24( quint32 i )
{
#if Q_BYTE_ORDER == Q_BIG_ENDIAN
    return i & 0xffffff;
#else
    int r = qFromBigEndian( i );
    return r >> 8;
#endif
}

U8::U8( bool initialize, int type, const QStringList &names )
{
    ok = false;
    isLz77 = false;
    paths.clear();
    nestedU8s.clear();
    fst = NULL;
    fstSize = 0;
    headerType = type;
    imetNames = names;
    if( !initialize )
		return;

    //headerType = type;
    //imetNames = names;

    ok = CreateEmptyData();
}

bool U8::CreateEmptyData()
{
    isLz77 = false;
    nestedU8s.clear();
    data = QByteArray( 0x20, '\0' );

    //create a generic u8 with no real info in it.  only a fst with a root node
    QBuffer buf( &data );
    if( !buf.open( QBuffer::WriteOnly ) )
    {
		qWarning() << "Can't create buffer";
		data.clear();
		return false;
    }
    buf.putChar( 'U' );
    buf.putChar( '\xAA' );
    buf.putChar( '8' );
    buf.putChar( '\x2D' );

    rootnode_offset = 0x00000020;
    quint32 tmp = qFromBigEndian( rootnode_offset );
    buf.write( (const char*)&tmp, 4 );

    fstSize = 0x0000000D;
    tmp = qFromBigEndian( fstSize );
    buf.write( (const char*)&tmp, 4 );

    data_offset = qFromBigEndian( 0x00000040 );
    tmp = qFromBigEndian( data_offset );
    buf.write( (const char*)&tmp, 4 );
    buf.close();

    //create the new fst
    QByteArray fstArray = QByteArray( 0x20, '\0' );
    fst = (FEntry*)( fstArray.data() );
    fst[ 0 ].Type = 1;
    fst[ 0 ].FileLength = qFromBigEndian( 0x00000001 );
    NameOff = 0x0C;

    data.append( fstArray );

    //point the fst pointer to the new data
    fst = (FEntry*)( data.data() + rootnode_offset );


    ok = true;
    return true;
}

bool U8::RenameEntry( const QString &path, const QString &newName )
{
    qDebug() << "U8::RenameEntry(" << path << "," << newName << ")";
    if( !ok )
    {
		qWarning() << "U8::RenameEntry -> archive has no data";
		return false;
    }
    if( newName.contains( "/" ) )
    {
		qWarning() << "U8::RenameEntry -> newName cannot contain \'/\'";
		return false;
    }
    //check if this is a path to a file in a nested archive
    QMap<QString, U8 >::iterator i = nestedU8s.begin();
    while( i != nestedU8s.constEnd() )
    {
		if( path.startsWith( i.key() ) && path != i.key() )
		{
			QString subPath = path;
			subPath.remove( 0, i.key().size() + 1 );//remove the path of the archive itself + the slash

			bool ret = i.value().RenameEntry( subPath, newName );
			if( !ret )
			{
				qWarning() << "U8::RenameEntry -> error replacing data in child U8 archive";
				return false;
			}
			//qDebug() << "child entry updated, now replacing child archive in this one";
			return ReplaceEntry( i.key(),i.value().GetData() );
			//NOTE - after replacing the entry, "i" is no longer valid.  keep that in mind when changing this code.  its a bitch to track down this bug
		}
		i++;
    }
    //make sure the new filename doesnt already exist
    QString parentPath = path;
    while( parentPath.startsWith( "/") )//remove leading "/"s
		parentPath.remove( 0, 1 );
    while( parentPath.endsWith( "/") )//remove trailing "/"s
		parentPath.resize( parentPath.size() - 1 );

    int slash = parentPath.lastIndexOf( "/" );
    if( slash != -1 )
    {
		parentPath.chop( ( parentPath.size() - slash ) );
		parentPath += "/" + newName;
    }
    else
		parentPath = newName;

    if( FindEntry( parentPath ) != -1 )
    {
		qWarning() << "U8::RenameEntry ->" << parentPath << "already exists in the archive";
		return false;
    }

    //find the entry to rename
    int entryToRename = FindEntry( path );
    if( entryToRename == -1 )
    {
		qWarning() << "U8::RenameEntry" << path << "doesn\'t exists in the archive";
		qWarning() << "choices are" << paths;
		return false;
    }

    quint32 newNameLen = newName.size();
    quint32 oldNameLen = FstName( entryToRename ).size();
    int difference = newNameLen - oldNameLen;


    quint32 nFstSize = fstSize + difference;
    int dataAdjustment = 0;
	if( RU( nFstSize, U8_HEADER_ALIGNMENT ) < RU( fstSize, U8_HEADER_ALIGNMENT ) )
		dataAdjustment = - RU( ( fstSize - nFstSize ), U8_HEADER_ALIGNMENT );

	else if( RU( nFstSize, U8_HEADER_ALIGNMENT ) > RU( fstSize, U8_HEADER_ALIGNMENT ) )
		dataAdjustment = RU( ( nFstSize - fstSize ), U8_HEADER_ALIGNMENT );

    qDebug() << "old size:" << hex << oldNameLen\
			<< "new size:" << hex << newNameLen\
			<< "difference:" << hex << difference
			<< "dataAdjustment:" << hex << dataAdjustment;
    QByteArray nFstData( ( qFromBigEndian( fst[ 0 ].FileLength ) ) * 0xc, '\0' );
    FEntry *nfst = (FEntry*)( nFstData.data() );
    //make the new root entry
    nfst[ 0 ] = fst[ 0 ];



    //the new data payload ( just copy the existing one )
    QByteArray nPayload = data.right( data.size() - data_offset );
    QByteArray nNameTable;
    nNameTable.append( '\0' );//add a null byte for the root name

    quint32 cnt = qFromBigEndian( fst[ 0 ].FileLength );//the number of entries


    for( quint32 i = 1; i < cnt; i++ )
    {
		FEntry *e = &fst[ i ];//old entry
		FEntry *ne = &nfst[ i ];//new entry

		ne->NameOffset = swap24( nNameTable.size() );//offset to the new entry's name in the string table

		//add the name to the new name table
		if( i == (quint32)entryToRename )
		{
			nNameTable.append( newName.toLatin1() );
		}
		else
			nNameTable.append( FstName( e ).toLatin1() );

		nNameTable.append( '\0' );
		if( e->Type )//nothing special to change for directories except the name offset
		{
			ne->Type = 1;
			ne->ParentOffset = e->ParentOffset;
			ne->NextOffset = e->NextOffset;
		}
		else//only need to change the name offset & the data offset for files
		{
			ne->Type = 0;
			ne->FileOffset = qFromBigEndian( qFromBigEndian( e->FileOffset ) + dataAdjustment );// + qFromBigEndian( dataAdjustment );
		    //qFromBigEndian( (quint32)( 0x20 + RU( U8_HEADER_ALIGNMENT, nFstSize ) + nPayload.size() ) );
			ne->FileLength = e->FileLength;
			qDebug() << "old offset" << hex << qFromBigEndian( e->FileOffset ) << "new offset" << hex << qFromBigEndian( e->FileOffset ) + dataAdjustment;
		}

    }
    //now put all the parts together to make a new U8 archive
    nFstData.append( nNameTable );

    data = QByteArray( 0x20, '\0' );
    QBuffer buf( &data );
    if( !buf.open( QBuffer::WriteOnly ) )
    {
		qWarning() << "U8::AddEntry -> Can't create buffer";
		return -1;
    }
    buf.putChar( 'U' );
    buf.putChar( '\xAA' );
    buf.putChar( '8' );
    buf.putChar( '\x2D' );

    rootnode_offset = 0x00000020;
    quint32 t = qFromBigEndian( rootnode_offset );
    buf.write( (const char*)&t, 4 );

    fstSize = nFstData.size();
    t = qFromBigEndian( fstSize );
    buf.write( (const char*)&t, 4 );

	data_offset = RU( ( 0x20 + fstSize ), U8_HEADER_ALIGNMENT );
    t = qFromBigEndian( data_offset );
    buf.write( (const char*)&t, 4 );
    buf.close();

    data.append( nFstData );//add the fst
    int padding = data_offset - data.size();//pad to 'data_offset' after the fst
    if( padding )
    {
		data.append( QByteArray( padding, '\0' ) );
    }
    data.append( nPayload );//add the actual file data

	padding = RU( data.size(), 0x20 ) - data.size();//pad the entire thing to 0x20 bytes TOTO: should probably already be done, and this step is not really necessary
    if( padding )
    {
		data.append( QByteArray( padding, '\0' ) );
    }



    CreateEntryList();

    //hexdump( data, 0, 0x100 );
    //hexdump( data );
    return true;
}

bool U8::ReplaceEntry( const QString &path, const QByteArray &nba, bool autoCompress )
{
    //qDebug() << "U8::ReplaceEntry(" << path << ")";
    if( !ok )
    {
		qWarning() << "U8::ReplaceEntry -> archive has no data";
		return false;
    }
    //check if this is a path to a file in a nested archive
    QMap<QString, U8 >::iterator i = nestedU8s.begin();
    while( i != nestedU8s.constEnd() )
    {
		if( path.startsWith( i.key() ) && path != i.key() )
		{
			QString subPath = path;
			subPath.remove( 0, i.key().size() + 1 );//remove the path of the archive itself + the slash
			//qDebug() << "replacing file" << subPath << "in child U8" << i.key();

			U8 ch = i.value();
			QString chPath = i.key();
			//qDebug() << "replacing" << subPath << "in child archive";
			bool ret = ch.ReplaceEntry( subPath, nba, autoCompress );
			if( !ret )
			{
				qWarning() << "U8::ReplaceEntry -> error replacing data in child U8 archive";
				return false;
			}
			//qDebug() << "child entry updated, now replacing child archive in this one";
			return ReplaceEntry( chPath, ch.GetData() );
			//NOTE - after replacing the entry, "i" is no longer valid.  keep that in mind when changing this code.  its a bitch to track down this bug
		}
		i++;
    }
    //find the entry to replace
    int entryToReplace = FindEntry( path );
    if( entryToReplace == -1 )
    {
		qWarning() << "U8::ReplaceEntry" << path << "doesn\'t exists in the archive";
		qWarning() << "choices are" << paths;
		return false;
    }
    if( qFromBigEndian( fst[ entryToReplace ].Type ) )
    {
		qWarning() << "U8::ReplaceEntry -> replacing a directory is not supported" << path;
		return false;
    }


    /*qDebug() << "old size:" << hex << oldSizePadded\
	    << "new size:" << hex << newSizePadded\
	    << "difference:" << hex << difference;*/
    QByteArray newData = nba;
    if( autoCompress )
    {
		QByteArray oldData = data.mid( qFromBigEndian( fst[ entryToReplace ].FileOffset ), qFromBigEndian( fst[ entryToReplace ].FileLength ) );
		bool oldCompressed = ( LZ77::GetLz77Offset( oldData ) > -1 || IsAshCompressed( oldData ) );
		if( oldCompressed && LZ77::GetLz77Offset( newData ) == -1 && !IsAshCompressed( newData ) )
		{
			newData = LZ77::Compress( newData );
		}
    }

	quint32 newSizePadded = RU( newData.size(), 0x20 );
	quint32 oldSizePadded = RU( qFromBigEndian( fst[ entryToReplace ].FileLength ), 0x20 );
    int difference = newSizePadded - oldSizePadded;

    data.remove( qFromBigEndian( fst[ entryToReplace ].FileOffset ), oldSizePadded );
    QByteArray newPaddedData = newData;
    if( newSizePadded > (quint32)newPaddedData.size() )
    {
		newPaddedData.append( QByteArray( newSizePadded - newPaddedData.size(), '\0' ) );
    }
    data.insert( qFromBigEndian( fst[ entryToReplace ].FileOffset ), newPaddedData );

    if( newSizePadded == oldSizePadded )//the sized match, so nothing else to change
		return true;

    //point the fst to the new data
    fst = (FEntry*)( data.data() + rootnode_offset );

    //edit the changed size
    FEntry *e = &fst[ entryToReplace ];
    e->FileLength = qFromBigEndian( newData.size() );
    //hexdump( (const void*)e, sizeof( FEntry ) );

    //edit all the file offsets after the changed item
    quint32 cnt = qFromBigEndian( fst[ 0 ].FileLength );
    for( quint32 i = entryToReplace + 1; i < cnt; i++ )
    {
		FEntry *e = &fst[ i ];//old entry
		if( e->Type )//nothing changes for directories
			continue;

		//qDebug() << "changed" << FstName( e ) << "offset from" << hex << qFromBigEndian( fst[ i ].FileOffset ) << "to" << qFromBigEndian( fst[ i ].FileOffset ) + difference;
		e->FileOffset = qFromBigEndian( qFromBigEndian( fst[ i ].FileOffset ) + difference );
    }
    CreateEntryList();

    //hexdump( data, 0, 0x100 );
    //hexdump( data );
    return true;

}

bool U8::RemoveEntry( const QString &path )
{
    //qDebug() << "U8::RemoveEntry(" << path << ")";
    if( !ok )
    {
		qWarning() << "U8::RemoveEntry -> archive has no data";
		return false;
    }
    //check if this is a path to a file in a nested archive
    QMap<QString, U8 >::iterator i = nestedU8s.begin();
    while( i != nestedU8s.constEnd() )
    {
		if( path.startsWith( i.key() ) && path != i.key() )
		{
			QString subPath = path;
			subPath.remove( 0, i.key().size() + 1 );//remove the path of the archive itself + the slash

			U8 ch = i.value();
			QString chPath = i.key();
			//qDebug() << "deleting" << subPath << "in child archive";

			bool ret = ch.RemoveEntry( subPath );
			if( !ret )
			{
				qWarning() << "U8::RemoveEntry -> error replacing data in child U8 archive";
				return false;
			}
			return ReplaceEntry( chPath, ch.GetData() );//insert the new nested archive in this one
			//NOTE - after replacing the entry, "i" is no longer valid.  keep that in mind when changing this code.  its a bitch to track down this bug
		}
		i++;
    }
    //find the entry to delete
    int entryToDelete = FindEntry( path );
    if( entryToDelete == -1 )
    {
		qWarning() << "U8::RemoveEntry" << path << "doesn\'t exists in the archive";
		return false;
    }

    //create a list of all the directory indexes that my have their size changed
    QList<int> parents;
    int parent;
    quint32 numDeletedEntries;
    quint32 fstLenToDelete;
    if( fst[ entryToDelete ].Type )//deleting a directory
    {
		parent = qFromBigEndian( fst[ entryToDelete ].ParentOffset );
		while( parent )
		{
			parents << parent;
			parent = qFromBigEndian( fst[ parent ].ParentOffset );
		}
		numDeletedEntries = qFromBigEndian( fst[ entryToDelete ].NextOffset ) - entryToDelete;
		fstLenToDelete = 0;
		for( quint32 i = entryToDelete; i < qFromBigEndian( fst[ entryToDelete ].NextOffset ); i ++ )
		{
			fstLenToDelete += FstName( i ).size() + 1 + 0xc;
		}

    }
    else//deleting a file
    {
		parent = entryToDelete - 1;
		while( parent )
		{
			if( fst[ parent ].Type && qFromBigEndian( fst[ parent ].NextOffset ) > (quint32)entryToDelete )
			{
				parents << parent;
				parent = qFromBigEndian( fst[ parent ].ParentOffset );
			}
			else
				parent--;
		}
		numDeletedEntries = 1;
		fstLenToDelete = FstName( entryToDelete ).size() + 1 + 0xc;

    }


    //qDebug() << "will delete" << numDeletedEntries << "entries";

    quint32 nFstSize = fstSize - fstLenToDelete;//old fst size - 0xc bytes for each deleted entry and the length of its name + 1 null byte
    //qDebug() << "nFstSize" << nFstSize;
    QByteArray nFstData( ( qFromBigEndian( fst[ 0 ].FileLength ) - numDeletedEntries ) * 0xc, '\0' );
    FEntry *nfst = (FEntry*)( nFstData.data() );

    //the new data payload ( all the files )
    QByteArray nPayload;
    QByteArray nNameTable;
    nNameTable.append( '\0' );//add a null byte for the root name

    quint32 cnt = qFromBigEndian( fst[ 0 ].FileLength );//the number of entries + 1 for the new one

    //make the new root entry
    nfst[ 0 ] = fst[ 0 ];
    nfst[ 0 ].FileLength = qFromBigEndian( cnt - numDeletedEntries );

    QList<int> movedDirs;//keep a list of the directories that are shifted back so their children can be shifted too
    for( quint32 i = 1; i < cnt; i++ )
    {
		if( i >= (quint32)entryToDelete && i < (quint32)( entryToDelete + numDeletedEntries ) )//skip deleted entries
			continue;

		//copy old entries and adjust as needed
		quint8 adj = i < (quint32)entryToDelete ? 0 : numDeletedEntries;
		quint32 ni = i - adj;

		//qDebug() << "keeping" << FstName( i ) << "in the new archive ( moved from" << hex << i << "to" << hex << ni << ")";
		//if( parents.contains( i ) )
	    //qDebug() << "\tthis is a parent of the deleted item";

		FEntry *e = &fst[ i ];//old entry
		FEntry *ne = &nfst[ ni ];//new entry

		nfst[ ni ].NameOffset = swap24( nNameTable.size() );//offset to the new entry's name in the string table

		//add the name to the new name table
		nNameTable.append( FstName( e ).toLatin1() );
		nNameTable.append( '\0' );


		if( e->Type )//directory
		{
			ne->Type = 1;
			if( !adj )//directories before the new entry
			{
				ne->ParentOffset = e->ParentOffset;
				ne->NextOffset = qFromBigEndian( qFromBigEndian( e->NextOffset ) \
												 - ( parents.contains( i ) ? numDeletedEntries : 0 ) );
			}
			else//directories after the new entry
			{
				ne->ParentOffset = qFromBigEndian( qFromBigEndian( e->ParentOffset ) \
												   - ( movedDirs.contains( qFromBigEndian( e->ParentOffset ) ) ? numDeletedEntries : 0 ) );
				ne->NextOffset = qFromBigEndian( qFromBigEndian( e->NextOffset ) - numDeletedEntries );

				movedDirs << i;

				//qDebug() << "e.parent:" << hex << qFromBigEndian( e->ParentOffset ) << "movedDirs:" << movedDirs;
				//hexdump( (const void*)ne, sizeof( FEntry) );
			}
		}
		else//file
		{
			ne->Type = 0;
			ne->FileOffset = \
							 qFromBigEndian( (quint32)( 0x20 + RU( nFstSize, U8_HEADER_ALIGNMENT ) + nPayload.size() ) );
			ne->FileLength = e->FileLength;
			nPayload.append( data.mid( qFromBigEndian( e->FileOffset ), qFromBigEndian( e->FileLength ) ) );
			int padding = RU( nPayload.size(), 0x20 ) - nPayload.size();//pad to 0x20 bytes between files
			if( padding )
			{
				nPayload.append( QByteArray( padding, '\0' ) );
			}
			//qDebug() << "writing fileOffset of" << hex << ni << hex << (quint32)( 0x20 + RU( U8_HEADER_ALIGNMENT, nFstSize ) + nPayload.size() );
		}
		//hexdump( (const void*)ne, sizeof( FEntry) );

    }
    //hexdump( nFstData );

    //now put all the parts together to make a new U8 archive
    nFstData.append( nNameTable );

    data = QByteArray( 0x20, '\0' );
    QBuffer buf( &data );
    if( !buf.open( QBuffer::WriteOnly ) )
    {
		qWarning() << "U8::AddEntry -> Can't create buffer";
		return -1;
    }
    buf.putChar( 'U' );
    buf.putChar( '\xAA' );
    buf.putChar( '8' );
    buf.putChar( '\x2D' );

    rootnode_offset = 0x00000020;
    quint32 t = qFromBigEndian( rootnode_offset );
    buf.write( (const char*)&t, 4 );

    fstSize = nFstData.size();
    t = qFromBigEndian( fstSize );
    buf.write( (const char*)&t, 4 );

	data_offset = RU( ( 0x20 + fstSize ), U8_HEADER_ALIGNMENT );
    t = qFromBigEndian( data_offset );
    buf.write( (const char*)&t, 4 );
    buf.close();

    data.append( nFstData );//add the fst
    int padding = data_offset - data.size();//pad to 'data_offset' after the fst
    if( padding )
    {
		data.append( QByteArray( padding, '\0' ) );
    }
    data.append( nPayload );//add the actual file data

	padding = RU( data.size(), 0x20 ) - data.size();//pad the entire thing to 0x20 bytes TOTO: should probably already be done, and this step is not really necessary
    if( padding )
    {
		data.append( QByteArray( padding, '\0' ) );
    }

    //make sure the pirvate variables are correct
    fst = (FEntry*)( data.data() + rootnode_offset );
    NameOff = 0x0C * qFromBigEndian( fst[ 0 ].FileLength );

    CreateEntryList();

    //hexdump( data );
    //qDebug() << "dataSize after removal:" << hex << data.size();

    return true;

}

int U8::AddEntry( const QString &path, int type, const QByteArray &newData )
{
    //qDebug() << "U8::AddEntry(" << path << "," << type << "," << hex << newData.size() << ")";
    //make sure there is actually data to manipulate
    if( !ok && !CreateEmptyData() )
    {
		return -1;
    }
    //check if this is a path to a file in a nested archive
    QMap<QString, U8 >::iterator i = nestedU8s.begin();
    while( i != nestedU8s.constEnd() )
    {
		if( path.startsWith( i.key() ) && path != i.key() )
		{
			QString subPath = path;
			//make a copy of the string, as "i" will become invalid on replace entry, and create a crash that is a bitch to track down
			QString thisPath = i.key();
			U8 chVal = i.value();
			//qDebug() << "adding entry" << subPath << "in child U8" << thisPath;
			subPath.remove( 0, thisPath.size() + 1 );//remove the path of the archive itself + the slash
			bool ret = chVal.AddEntry( subPath, type, newData );
			if( !ret )
			{
				qWarning() << "U8::AddEntry -> error replacing data in child U8 archive";
				return false;
			}

			if( !ReplaceEntry( thisPath, chVal.GetData() ) )//insert the new nested archive in this one
				return -1;

			//qDebug() << "done replacing the child entry in this archive, finding the index to return (" << thisPath << ")";
			return FindEntry( thisPath );//just return the index of the nested archive
		}
		i++;
    }

    //make sure this path doesnt already exist
    if( FindEntry( path ) != -1 )
    {
		qWarning() << "U8::AddEntry" << path << "already exists in the archive";
		return -1;
    }
    //find the parent for this new entry
    QString parentPath = path;
    while( parentPath.startsWith( "/") )//remove leading "/"s
		parentPath.remove( 0, 1 );
    while( parentPath.endsWith( "/") )//remove trailing "/"s
		parentPath.resize( parentPath.size() - 1 );

    int parent = 0;
    int slash = parentPath.lastIndexOf( "/" );
    if( slash != -1 )
    {
		//parentPath.resize( ( parentPath.size() - slash ) - 2 );
		parentPath.chop( ( parentPath.size() - slash ) );
		parent = FindEntry( parentPath );
		if( parent == -1 )//need to create subfolders for this fucker
		{
			parent = AddEntry( parentPath, 1 );
			if( parent == -1 )
			{
				qWarning() << "U8::AddEntry -> could not create subfolders for" << path;
				return -1;
			}
		}
		if( !fst[ parent ].Type )//parent maps to a file, not folder
		{
			qWarning() << "U8::AddEntry -> can't add child entry to a file";
			return -1;
		}
    }
    //qDebug() << "adding entry to" << FstName( parent );

    //find the index to insert the new path
    int newEntry = qFromBigEndian( fst[ parent ].NextOffset );

    //create a list of all the directory indexes that my have their size changed to hold the new entry
    QList<int> parents;
    while( parentPath.contains( "/" ) )
    {
		parents << FindEntry( parentPath );
		int slash = parentPath.lastIndexOf( "/" );
		parentPath.chop( ( parentPath.size() - slash ) );
    }
    parents << FindEntry( parentPath );
    //qDebug() << "parents" << parents;

    //create the new data

    //the new fst
    QString name = path;//get the actual name of this entry
    int sl = name.lastIndexOf( "/" );
    if( sl != -1 )
    {
		name.remove( 0, sl + 1 );
    }

    quint32 nFstSize = fstSize + 0x0C + name.size() + 1;//old fst size + 0xc bytes for the new entry + room for the new name in the string table
    QByteArray nFstData( ( qFromBigEndian( fst[ 0 ].FileLength ) + 1 ) * 0xc, '\0' );
    FEntry *nfst = (FEntry*)( nFstData.data() );

    //the new data payload ( all the files )
    QByteArray nPayload;
    QByteArray nNameTable;
    nNameTable.append( '\0' );//add a null byte for the root name

    quint32 cnt = qFromBigEndian( fst[ 0 ].FileLength ) + 1;//the number of entries + 1 for the new one

    //make the new root entry
    nfst[ 0 ] = fst[ 0 ];
    nfst[ 0 ].FileLength = qFromBigEndian( cnt );

    QList<int> movedDirs;//keep a list of the directories that are shifted back so their children can be shifted too
    for( quint32 i = 1; i < cnt; i++ )
    {
		nfst[ i ].NameOffset = swap24( nNameTable.size() );//offset to the new entry's name in the string table
		if( i == (quint32)newEntry )//add the new entry
		{
			FEntry *ne = &nfst[ i ];
			if( type )
			{
				ne->Type = 1;
				ne->ParentOffset = qFromBigEndian( parent );
				ne->NextOffset = qFromBigEndian( i + 1 );
			}
			else
			{
				ne->Type = 0;
				ne->FileLength = qFromBigEndian( (quint32)newData.size() );
				ne->FileOffset =\
								qFromBigEndian( (quint32)( 0x20 + RU( nFstSize, U8_HEADER_ALIGNMENT ) + nPayload.size() ) );
				nPayload.append( newData );
				int padding = RU( nPayload.size(), 0x20 ) - nPayload.size();//pad to 0x20 bytes between files
				if( padding )
				{
					nPayload.append( QByteArray( padding, '\0' ) );
				}
			}
			nNameTable.append( name.toLatin1() );//add this entry's name to the table
			nNameTable.append( '\0' );
			continue;
		}

		//copy old entries and adjust as needed
		quint8 adj = i < (quint32)newEntry ? 0 : 1;
		quint32 ni = i - adj;

		FEntry *e = &fst[ ni ];//old entry
		FEntry *ne = &nfst[ i ];//new entry

		//add the name to the new name table
		nNameTable.append( FstName( e ).toLatin1() );
		nNameTable.append( '\0' );
		if( e->Type )
		{
			ne->Type = 1;
			if( !adj )//directories before the new entry
			{
				ne->ParentOffset = e->ParentOffset;
				ne->NextOffset = qFromBigEndian( qFromBigEndian( e->NextOffset ) \
												 + ( parents.contains( i ) ? 1 : 0 ) );
			}
			else//directories after the new entry
			{
				movedDirs << ni;
				ne->ParentOffset = qFromBigEndian( qFromBigEndian( e->ParentOffset ) \
												   + ( movedDirs.contains( qFromBigEndian( e->ParentOffset ) ) ? 1 : 0 ) );
				ne->NextOffset = qFromBigEndian( qFromBigEndian( e->NextOffset ) + 1 );
			}
		}
		else//all file entries
		{
			ne->Type = 0;
			ne->FileOffset = \
							 qFromBigEndian( (quint32)( 0x20 + RU( nFstSize, U8_HEADER_ALIGNMENT ) + nPayload.size() ) );
			ne->FileLength = e->FileLength;
			nPayload.append( data.mid( qFromBigEndian( e->FileOffset ), qFromBigEndian( e->FileLength ) ) );
			int padding = RU( nPayload.size(), 0x20 ) - nPayload.size();//pad to 0x20 bytes between files
			if( padding )
			{
				nPayload.append( QByteArray( padding, '\0' ) );
			}
		}

    }
    //now put all the parts together to make a new U8 archive
    nFstData.append( nNameTable );

    data = QByteArray( 0x20, '\0' );
    QBuffer buf( &data );
    if( !buf.open( QBuffer::WriteOnly ) )
    {
		qWarning() << "U8::AddEntry -> Can't create buffer";
		return -1;
    }
    buf.putChar( 'U' );
    buf.putChar( '\xAA' );
    buf.putChar( '8' );
    buf.putChar( '\x2D' );

    rootnode_offset = 0x00000020;
    quint32 t = qFromBigEndian( rootnode_offset );
    buf.write( (const char*)&t, 4 );

    fstSize = nFstData.size();
    t = qFromBigEndian( fstSize );
    buf.write( (const char*)&t, 4 );

	data_offset = RU( ( 0x20 + fstSize ), U8_HEADER_ALIGNMENT );
    t = qFromBigEndian( data_offset );
    buf.write( (const char*)&t, 4 );
    buf.close();

    data.append( nFstData );//add the fst
    int padding = data_offset - data.size();//pad to 'data_offset' after the fst
    if( padding )
    {
		data.append( QByteArray( padding, '\0' ) );
    }
    data.append( nPayload );//add the actual file data

	padding = RU( data.size(), 0x20 ) - data.size();//pad the entire thing to 0x20 bytes TOTO: should probably already be done, and this step is not really necessary
    if( padding )
    {
		data.append( QByteArray( padding, '\0' ) );
    }

    //make sure the pirvate variables are correct
    fst = (FEntry*)( data.data() + rootnode_offset );
    NameOff = 0x0C * qFromBigEndian( fst[ 0 ].FileLength );
    //qDebug() << "done adding the entry";

    CreateEntryList();

    //hexdump( data );
    return newEntry;
}

void U8::CreateEntryList()
{
    //qDebug() << "U8::CreateEntryList";
    paths.clear();
    nestedU8s.clear();
    fst = (FEntry*)( data.data() + rootnode_offset );
    quint32 cnt = qFromBigEndian( fst[ 0 ].FileLength );
    if( ( cnt * 0x0d ) + rootnode_offset > (quint32)data.size() )
    {
		qWarning() << "U8::CreateEntryList -> invalid archive";
		ok = false;
		data.clear();
		paths.clear();
		fst = NULL;
		return;
    }
    NameOff = cnt * 0x0C;
    bool fixWarn = false;//ony print the warning 1 time

    //qDebug() << "cnt" << hex << cnt;
    for( quint32 i = 1; i < cnt; ++i )//this is not the most effecient way to do things, but it seems to work ok and these archives are small enough that it happens fast anyways
    {
		//start at the beginning of the fst and enter every directory whos "nextoffset" is greater than this index,
		//adding the names along the way and hopefully finding "i"
		QString path;
		quint32 current = 1;
		quint32 folder = 0;
		while( 1 )
		{
			if( !current )
			{
				qWarning() << "U8::CreateEntryList -> error parsing the archive";
				break;
			}
			if( current == i )
			{
				path += FstName( i );
				if( fst[ current ].Type )
				{
					//sanity check:  make sure the parent offset is correct
					if( folder != qFromBigEndian( fst[ current ].ParentOffset ) )
					{
						qWarning() << "U8::CreateEntryList -> error parsing the archive - recursion mismatch in"
								<< path << "expected:" << hex << folder << "got:" << hex << qFromBigEndian( fst[ current ].ParentOffset )\
								<< "(" << FstName( qFromBigEndian( fst[ current ].ParentOffset ) ) << ")";

						//some tools use "recursion" instead of "parent offset".
						//it just coincidentally works for archives such as the opening.bnr and .app since they dont have many folders to mess up
						if( qFromBigEndian( fst[ current ].ParentOffset ) == (quint32)path.count( "/" ) )
						{
							if( !fixWarn )
							{
								qWarning() << "This archive was made by a broken tool such as U8Mii.  I'm trying to fix it, but I can't make any promises";
								fixWarn = true;
							}
							fst[ current ].ParentOffset = qFromBigEndian( folder );
						}
					}
					path += "/";
					paths << path;
				}
				else
				{
					//add this file to the list of entries
					paths << path;

					//check if this file is really a U8 archive, and add it to the list of children
					QByteArray chData = data.mid( qFromBigEndian( fst[ current ].FileOffset ), qFromBigEndian( fst[ current ].FileLength ) );
					if( IsU8( chData ) )
					{
						U8 child( chData );
						if( child.IsOK() )
						{
							nestedU8s.insert( path, child );
							foreach( QString chPath, child.Entries() )
							{
								QString newPath = path + "/" + chPath;
								paths << newPath;
							}
						}
					}
				}
				break;
			}
			if( fst[ current ].Type && i < qFromBigEndian( fst[ current ].NextOffset ) )
			{
				path += FstName( current ) + "/";
				folder = current;
				current++;
				continue;

			}
			current = NextEntryInFolder( current, folder );
		}
		//paths << path;
    }
    //qDebug() << "done";
    //qDebug() << "paths" << paths;
}

U8::U8( const QByteArray &ba )
{
    //qDebug() << "U8::U8 dataSize:" << hex << ba.size();
    Load( ba );
}

void U8::Load( const QByteArray &ba )
{
    ok = false;
    isLz77 = false;
    headerType = U8_Hdr_none;
    paths.clear();
    imetNames.clear();
    /*if( ba.size() < 0x80 )
    {
	//qWarning() << "U8::Load:" << hex << ba.size();
	//qWarning() << "U8::Load -> where is the rest of the data?";
	return;
    }*/
    data = ba;
    if( IsAshCompressed( data ) )//decrypt ASH0 files
		data = DecryptAsh( data );

    quint32 tmp;
    int off = LZ77::GetLz77Offset( data );
    int off2 = GetU8Offset( data );
    if( off != -1 && ( off2 == -1 || ( off2 != -1 && off < off2 ) ) )
    {
		isLz77 = true;
		data = LZ77::Decompress( data );
		off2 = GetU8Offset( data );
    }

    if( off2 == -1 )
    {
		qDebug() << "This isnt a U8";
		return;
    }

    data = data.mid( off2 );

    QBuffer buf( &data );
    if( !buf.open( QBuffer::ReadOnly ) )
    {
		qWarning() << "Can't create buffer";
		return;
    }

    buf.seek( 4 );
    buf.read( ( char*)&tmp, 4 );
    rootnode_offset = qFromBigEndian( tmp );
    if( rootnode_offset != 0x20 )
    {
		qWarning() << "rootnodeOffset" << hex << rootnode_offset;
		qWarning() << hex << data.size();
		hexdump( data );
    }

    buf.read( ( char*)&tmp, 4 );
    fstSize = qFromBigEndian( tmp );

    buf.read( ( char*)&tmp, 4 );
    data_offset = qFromBigEndian( tmp );

    buf.close();

    CreateEntryList();


    ReadHeader( ba );
    ok = true;
    //qDebug() << "loaded" << paths;
}

bool U8::IsOK()
{
    return ok;
}

QString U8::FstName( const FEntry *entry )
{
    //qDebug() << "U8::FstName";
    if( entry == &fst[ 0 ] )
		return QString();

    int nameStart = swap24( entry->NameOffset ) + NameOff + rootnode_offset;
    return data.mid( nameStart, nameStart + 0x100 );
}

QString U8::FstName( quint32 i )
{
    if( i > fst[ 0 ].FileLength )
    {
		qWarning() << "U8::FstName -> index is out of range";
		return QString();
    }
    return FstName( &fst[ i ] );
}

quint32 U8::NextEntryInFolder( quint32 current, quint32 directory )
{
    //qDebug() << "U8::NextEntryInFolder(" << hex << current << "," << hex << directory << ")";
    quint32 next = ( fst[ current ].Type ? qFromBigEndian( fst[ current ].FileLength ) : current + 1 );
    //qDebug() << "next" << next << "len" << hex << qFromBigEndian( fst[ directory ].FileLength );
    if( next < qFromBigEndian( fst[ directory ].FileLength ) )
		return next;

    return 0;
}

int U8::FindEntry( const QString &str, int d )
{
    //qDebug() << "U8::FindEntry(" << str << "," << d << ")";
    if( str.isEmpty() )
		return 0;

    if( str.startsWith( "/" ) )
    {
		QString r = str;
		r.remove( 0, 1 );
		return FindEntry( r, d );
    }

    if( !fst[ d ].Type )
    {
		qDebug() << "ERROR!!" << FstName( &fst[ d ] ) << "is not a directory";
		return -1;
    }

    int next = d + 1;
    FEntry *entry = &fst[ next ];

    while( next )
    {
		QString item;
		int sl = str.indexOf( "/" );
		if( sl > -1 )
			item = str.left( sl );
		else
			item = str;

		if( FstName( entry ) == item )
		{
			if( item == str || item + "/" == str )//this is the item we are looking for
			{
				return next;
			}
			//this item is a parent folder of one we are looking for
			return FindEntry( str.right( ( str.size() - sl ) - 1 ), next );
		}
		//find the next entry in this folder
		next = NextEntryInFolder( next, d );
		entry = &fst[ next ];
    }

    //no entry with the given path was found
    return -1;
}

const QByteArray U8::GetData( const QString &str, bool onlyPayload )
{
    //qDebug() << "U8::GetData(" << str << ")";
    if( str.isEmpty() )//give the data for this whole archive
    {
		if( onlyPayload )
			return data;

		//qDebug() << "U8::GetData -> returning all the data for this archive.  headerType:" << headerType;

		QByteArray ret = data;

		switch( headerType )
		{
		case U8_Hdr_IMET_app:
		case U8_Hdr_IMET_bnr:
			ret = AddIMET( headerType );//data is lz77 compressed in this function if it needs to be
			break;
		case U8_Hdr_IMD5:
			{
				if( isLz77 )
					ret = LZ77::Compress( ret );

				ret = AddIMD5( ret );

				//hexdump( ret, 0, 0x40 );
			}
			break;
		default:
			break;
		}
		return ret;
    }
    //check if this is a path to a file in a nested archive
    QMap<QString, U8 >::iterator i = nestedU8s.begin();
    while( i != nestedU8s.constEnd() )
    {
		if( str.startsWith( i.key() ) && str != i.key() )
		{
			QString subPath = str;
			subPath.remove( 0, i.key().size() + 1 );//remove the path of the archive itself + the slash
			return i.value().GetData( subPath, onlyPayload );
		}
		i++;
    }
    int index = FindEntry( str );
    if( index < 0 )
    {
		qWarning() << "U8::GetData" << str << "was not found in the archive";
		return QByteArray();
    }
    if( fst[ index ].Type )
    {
		qWarning() << "U8::GetData" << str << "is a directory";
		return QByteArray();
    }
    QByteArray ret = data.mid( qFromBigEndian( fst[ index ].FileOffset ), qFromBigEndian( fst[ index ].FileLength ) );
    //hexdump( ret, 0, 0x40 );
    if( onlyPayload )
    {
		if( LZ77::GetLz77Offset( ret ) != -1 )
			ret = LZ77::Decompress( ret );

		else if( IsAshCompressed( ret ) )
			ret = DecryptAsh( ret );
    }
    return ret;
}

quint32 U8::GetSize( const QString &str )
{
    if( str.isEmpty() )
    {
		return data.size();
    }
    int index = FindEntry( str );
    if( index < 0 )
    {
		qWarning() << "U8::GetSize" << str << "was not found in the archive";
		return 0;
    }
    if( fst[ index ].Type )
    {
		qWarning() << "U8::GetSize" << str << "is a directory";
		return 0;
    }
    return  qFromBigEndian( fst[ index ].FileLength );
}

const QStringList U8::Entries()
{
    return paths;
}

int U8::GetU8Offset( const QByteArray &ba )
{
    QByteArray start = ba.left( 5000 );
    return start.indexOf( "U\xAA\x38\x2d" );
}

bool U8::IsU8( const QByteArray &ba )
{
    QByteArray data = ba;
    if( IsAshCompressed( data ) )//decrypt ASH0 files
		data = DecryptAsh( data );

    int off = LZ77::GetLz77Offset( data );//decrypt LZ77
    if( off != -1 )
		data = LZ77::Decompress( data );

    QByteArray start = data.left( 5000 );
    return start.indexOf( "U\xAA\x38\x2d" ) != -1;
}

#define IMET_MAX_NAME_LEN 0x2a
typedef struct
{
	quint32 sig; // "IMET"
	quint32 unk1;
	quint32 unk2;
	quint32 filesizes[ 3 ];
	quint32 unk3;
	quint16 names[ 10 ][ IMET_MAX_NAME_LEN ];
	quint8 zeroes2[ 0x24c ];
	quint8 md5[ 0x10 ];
} IMET;

void U8::ReadHeader( const QByteArray &ba )
{
    //qDebug() << "U8::ReadHeader(" << hex << ba.size() << ")";
    //hexdump( ba );
    headerType = U8_Hdr_none;
    imetNames.clear();
    if( ba.startsWith( "IMD5" ) )//dont bother to read any more data
    {
		//qDebug() << "IMD5 header";
		headerType = U8_Hdr_IMD5;
		return;
    }

    QByteArray start = ba.left( sizeof( IMET ) + 0x80 );
    QBuffer buf( &start );
    if( !buf.open( QBuffer::ReadOnly ) )
    {
		qWarning() << "U8::ReadHeader Can't create buffer";
		return;
    }
    int off = start.indexOf( "IMET" );
    //qDebug() << "imet offset" << hex << off << "u8 offset" << hex << GetU8Offset( ba );
    if( off == 0x40 || off == 0x80 )//read imet header
    {
		if( off > GetU8Offset( ba ) )//in case somebody wants to put a IMET archive inside another U8 for whatever reason
			return;

		if( off == 0x40 )
			headerType = U8_Hdr_IMET_bnr;
		else
			headerType = U8_Hdr_IMET_app;

		buf.seek( off );
		IMET imet;
		buf.read( (char*)&imet, sizeof( IMET ) );
		buf.close();

		for( int h = 0; h < 10; h ++ )
		{
			QString name;
			for( int i = 0; i < IMET_MAX_NAME_LEN && imet.names[ h ][ i ] != 0; i++ )//no need to switch endian, 0x0000 is a palendrome
				name += QChar( qFromBigEndian( imet.names[ h ][ i ] ) );

			imetNames << name;
		}
		//done
		return;
    }
}
QByteArray U8::AddIMD5( QByteArray ba )
{
    quint32 size = ba.size();

	/*MD5 hash;
    hash.update( ba.data(), size );
	hash.finalize();*/

    size = qFromBigEndian( size );

    QByteArray imd5;
    QBuffer buf( &imd5 );
    if( !buf.open( QBuffer::WriteOnly ) )
    {
		qWarning() << "U8::AddIMD5 Can't create buffer";
		return QByteArray();
    }
    buf.putChar( 'I' );
    buf.putChar( 'M' );
    buf.putChar( 'D' );
    buf.putChar( '5' );

    buf.write( (const char*)&size, 4 );
    buf.seek( 0x10 );

	buf.write( GetMd5( ba ) );
	//buf.write( (const char*)hash.hexdigestChar(), 16 );
    buf.close();
    //qDebug() << hash.hexdigest().c_str();
    //hexdump( (void*)imd5.data(), 0x20 );

    ba.prepend( imd5 );

    return ba;
}

const QByteArray U8::AddIMD5()
{
    QByteArray ba = data;
    ba = AddIMD5( ba );
    return ba;
}

QByteArray U8::GetIMET( const QStringList &names, int paddingType, quint32 iconSize, quint32 bannerSize, quint32 soundSize )
{
    QByteArray ret = QByteArray( sizeof( IMET ) + 0x40, '\0' );
    QBuffer buf( &ret );
    if( !buf.open( QBuffer::WriteOnly ) )
    {
		qWarning() << "U8::GetIMET Can't create buffer";
		return QByteArray();
    }
    buf.seek( 0 + 0x40 );
    buf.putChar( 'I' );
    buf.putChar( 'M' );
    buf.putChar( 'E' );
    buf.putChar( 'T' );
    buf.seek( 6 + 0x40 );
    buf.putChar( '\x6' );
    buf.seek( 11 + 0x40 );
    buf.putChar( '\x3' );

    quint32 tmp = qFromBigEndian( iconSize );
    buf.write( (char*)&tmp, 4 );
    tmp = qFromBigEndian( bannerSize );
    buf.write( (char*)&tmp, 4 );
    tmp = qFromBigEndian( soundSize );
    buf.write( (char*)&tmp, 4 );

    int nameOffset = 0x1c + 0x40;
    int numNames = names.size();
    for( int i = 0; i < numNames; i++ )
    {
		QString name = names.at( i );
		int nameLen = name.size();
		buf.seek( nameOffset + ( IMET_MAX_NAME_LEN * i * 2 ) );
		for( int j = 0; j < nameLen; j++ )
		{
			quint16 letter = qFromBigEndian( name.at( j ).unicode() );
			buf.write( ( const char*)&letter, 2 );
		}
    }

	/*MD5 hash;
    hash.update( ret.data(), sizeof( IMET ) + 0x40 );
	hash.finalize();*/

    buf.seek( 0x5f0 );
	buf.write( GetMd5( ret ) );
	//buf.write( (const char*)hash.hexdigestChar(), 16 );
    buf.close();

    switch( paddingType )
    {
    case U8_Hdr_IMET_app:
		ret.prepend( QByteArray( 0x40, '\0' ) );
		break;
    case U8_Hdr_IMET_bnr:
		break;
    default:
		ret.remove( 0, 0x40 );
		break;
    }
    return ret;
}

const QStringList U8::IMETNames()
{
    return imetNames;
}

void U8::SetImetNames( const QStringList &names )
{
    imetNames.clear();
    imetNames = names;
}

const QByteArray U8::AddIMET( int paddingType )
{
    QByteArray ret = GetData( "meta/icon.bin" );

    quint32 iconSize;
    quint32 bannerSize;
    quint32 soundSize;

    //if these are lz77 compressed, write the uncompressed size to the header  ?bannerbomb?
    //otherwise write the size - 0x20 ( imd5 header size )

    if( LZ77::GetLz77Offset( ret ) != -1 )
		iconSize = LZ77::GetDecompressedSize( ret );
    else
		iconSize = ret.size() - 0x20;

    ret = GetData( "meta/banner.bin" );
    if( LZ77::GetLz77Offset( ret ) != -1 )
		bannerSize = LZ77::GetDecompressedSize( ret );
    else
		bannerSize = ret.size() - 0x20;

    ret = GetData( "meta/sound.bin" );
    if( LZ77::GetLz77Offset( ret ) != -1 )
		soundSize = LZ77::GetDecompressedSize( ret );
    else
		soundSize = ret.size() - 0x20;

    ret = GetIMET( imetNames, paddingType, iconSize, bannerSize, soundSize );
    if( isLz77 )//really?  can the entire banner be lz77 compressed?
		ret += LZ77::Compress( data );
    else
		ret += data;

    return ret;
}

int U8::GetHeaderType()
{
    return headerType;
}