The-Homebrew-Cloud/JNUSLib
2
0
Fork 0
mirror of https://github.com/Maschell/JNUSLib.git synced 2024-06-27 03:16:04 +02:00
Code Issues Packages Projects Releases Wiki Activity

Refactor and fix FST parsing.

Browse Source 
- Fix a bug when parsing certain FST of system titles.
- Remove the Content reference on the FSTEntry. The Content can get be get via the TMD using getContentByIndex. To get the FSTInfo for the content use FSTUtils.getFSTInfoForContent.
- Children are added manually to a FSTEntry (getting ride of side effects).
- The NUSTitleLoader sets the FST to the dataprovider. The WUDDataProvider needs the FST to get access to the FSTContentInfo (=> calculating the offsets of the content files).
- Remove some unused functions from the FSTEntry class
- To get the FSTEntries for the certain content use getFSTEntriesByContentIndex
This commit is contained in:
Maschell 2019-04-26 10:08:09 +02:00
parent 157957f3ee
commit ddee4b9f5d
12 changed files with 78 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/de/mas/wiiu/jnus/DecryptionService.java
View File

@ -20,19 +20,16 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutionException;
import de.mas.wiiu.jnus.entities.content.Content;
import de.mas.wiiu.jnus.entities.fst.FSTEntry;
import de.mas.wiiu.jnus.interfaces.FSTDataProvider;
import de.mas.wiiu.jnus.utils.CheckSumWrongException;
import de.mas.wiiu.jnus.utils.FSTUtils;
import de.mas.wiiu.jnus.utils.FileUtils;
import de.mas.wiiu.jnus.utils.HashUtil;
import de.mas.wiiu.jnus.utils.Utils;
import lombok.val;
import lombok.extern.java.Log;
@ -66,7 +63,7 @@ public final class DecryptionService {
public CompletableFuture<Void> decryptFSTEntryToAsync(boolean useFullPath, FSTEntry entry, String outputPath, boolean skipExistingFile) {
return CompletableFuture.runAsync(() -> {
try {
if (entry.isNotInPackage() || !entry.getContent().isPresent()) {
if (entry.isNotInPackage()) {
return;
}
@ -99,18 +96,9 @@ public final class DecryptionService {
return;
}
if (targetFile.length() == entry.getFileSize()) {
Content c = entry.getContent().get();
if (c.isHashed()) {
log.info("File already exists: " + entry.getFilename());
return;
} else {
if (Arrays.equals(HashUtil.hashSHA1(target, (int) c.getDecryptedFileSize()), c.getSHA2Hash())) {
log.info("File already exists: " + entry.getFilename());
return;
} else {
log.info("File already exists with the same filesize, but the hash doesn't match: " + entry.getFilename());
}
}
log.info("File already exists: " + entry.getFilename());
return;
} else {
log.info("File already exists but the filesize doesn't match: " + entry.getFilename());

9
src/de/mas/wiiu/jnus/NUSTitleLoader.java
View File

@ -19,7 +19,6 @@ package de.mas.wiiu.jnus;
import java.io.IOException;
import java.io.InputStream;
import java.text.ParseException;
import java.util.Map;
import java.util.Optional;
import java.util.function.Supplier;
@ -84,11 +83,13 @@ public class NUSTitleLoader {
fstBytes = aesDecryption.decrypt(fstBytes);
}
Map<Integer, Content> contents = result.getTMD().getAllContents();
FST fst = FST.parseFST(fstBytes, contents);
FST fst = FST.parseFST(fstBytes);
result.setFST(Optional.of(fst));
// The dataprovider may need the FST to calculate the offset of a content
// on the partition.
dataProvider.setFST(fst);
return result;
}
}

2
src/de/mas/wiiu/jnus/entities/TMD.java
View File

@ -54,8 +54,6 @@ public final class TMD {
private static final int CERT1_LENGTH = 0x400;
private static final int CERT2_LENGTH = 0x300;
@Getter private final int signatureType; // 0x000
@Getter private final byte[] signature; // 0x004
@Getter private final byte[] issuer; // 0x140

16
src/de/mas/wiiu/jnus/entities/content/Content.java
View File

@ -50,10 +50,6 @@ public class Content implements Comparable<Content> {
@Getter private final long encryptedFileSize;
@Getter private final byte[] SHA2Hash;
@Getter private final List<FSTEntry> entries = new ArrayList<>();
@Getter @Setter private ContentFSTInfo contentFSTInfo;
private Content(ContentParam param) {
this.ID = param.getID();
this.index = param.getIndex();
@ -68,7 +64,7 @@ public class Content implements Comparable<Content> {
* @param input
* 0x30 byte of data from the TMD (starting at 0xB04)
* @return content object
* @throws ParseException
* @throws ParseException
*/
public static Content parseContent(byte[] input) throws ParseException {
if (input == null || input.length != CONTENT_SIZE) {
@ -128,16 +124,6 @@ public class Content implements Comparable<Content> {
return String.format("%08X%s", getID(), Settings.ENCRYPTED_CONTENT_EXTENTION);
}
/**
* Adds a content to the internal entry list.
*
* @param entry
* that will be added to the content list
*/
public void addEntry(FSTEntry entry) {
getEntries().add(entry);
}
/**
* Returns the size of the decrypted content.
*

11
src/de/mas/wiiu/jnus/entities/fst/FST.java
View File

@ -21,7 +21,6 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import de.mas.wiiu.jnus.entities.content.Content;
import de.mas.wiiu.jnus.entities.content.ContentFSTInfo;
import de.mas.wiiu.jnus.utils.ByteUtils;
import lombok.Getter;
@ -50,14 +49,12 @@ public final class FST {
*
* @param fstData
* raw decrypted FST data
* @param contentsMappedByIndex
* map of index/content
* @return
* @throws ParseException
* @throws ParseException
*/
public static FST parseFST(byte[] fstData, Map<Integer, Content> contentsMappedByIndex) throws ParseException {
public static FST parseFST(byte[] fstData) throws ParseException {
if (!Arrays.equals(Arrays.copyOfRange(fstData, 0, 3), new byte[] { 0x46, 0x53, 0x54 })) {
throw new ParseException("Failed to parse FST",0);
throw new ParseException("Failed to parse FST", 0);
}
int sectorSize = ByteUtils.getIntFromBytes(fstData, 0x04);
@ -94,7 +91,7 @@ public final class FST {
FSTEntry root = result.getRoot();
FSTService.parseFST(root, fstSection, nameSection, contentsMappedByIndex, contentFSTInfos, sectorSize);
FSTService.parseFST(root, fstSection, nameSection, sectorSize);
return result;
}

47
src/de/mas/wiiu/jnus/entities/fst/FSTEntry.java
View File

@ -16,7 +16,6 @@
****************************************************************************/
package de.mas.wiiu.jnus.entities.fst;
import java.io.File;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.List;
@ -50,31 +49,24 @@ public class FSTEntry {
@Getter private final long fileSize;
@Getter private final long fileOffset;
@Getter private final Optional<Content> content;
@Getter private final boolean isDir;
@Getter private final boolean isRoot;
@Getter private final boolean isNotInPackage;
@Getter private final short contentFSTID;
@Getter private final short contentIndex;
protected FSTEntry(FSTEntryParam fstParam) {
this.filenameSupplier = fstParam.getFileNameSupplier();
this.flags = fstParam.getFlags();
this.parent = fstParam.getParent();
if (parent.isPresent()) {
parent.get().children.add(this);
}
this.fileSize = fstParam.getFileSize();
this.fileOffset = fstParam.getFileOffset();
this.content = fstParam.getContent();
if (content.isPresent()) {
content.get().addEntry(this);
}
this.isDir = fstParam.isDir();
this.isRoot = fstParam.isRoot();
this.isNotInPackage = fstParam.isNotInPackage();
this.contentFSTID = fstParam.getContentFSTID();
this.contentIndex = fstParam.getContentIndex();
}
/**
@ -93,7 +85,6 @@ public class FSTEntry {
FSTEntryParam param = new FSTEntryParam();
param.setFileNameSupplier(() -> filename);
param.setFileSize(content.getDecryptedFileSize());
param.setContent(Optional.of(content));
param.setDir(false);
param.setParent(Optional.of(parent));
return new FSTEntry(param);
@ -158,26 +149,6 @@ public class FSTEntry {
return result;
}
public List<FSTEntry> getFSTEntriesByContent(Content content) {
List<FSTEntry> entries = new ArrayList<>();
if (this.content.isPresent() && this.content.get().equals(content)) {
entries.add(this);
}
for (FSTEntry child : getChildren()) {
entries.addAll(child.getFSTEntriesByContent(content));
}
return entries;
}
public long getFileOffsetBlock() {
if (getContent().isPresent() && getContent().get().isHashed()) {
return (getFileOffset() / 0xFC00) * 0x10000;
} else {
return getFileOffset();
}
}
public void printRecursive(int space) {
printRecursive(System.out, space);
}
@ -202,7 +173,7 @@ public class FSTEntry {
@Override
public String toString() {
return "FSTEntry [filename=" + getFilename() + ", path=" + getPath() + ", flags=" + flags + ", filesize=" + fileSize + ", fileoffset=" + fileOffset
+ ", content=" + content + ", isDir=" + isDir + ", isRoot=" + isRoot + ", notInPackage=" + isNotInPackage + "]";
+ ", isDir=" + isDir + ", isRoot=" + isRoot + ", notInPackage=" + isNotInPackage + "]";
}
@Data
@ -215,13 +186,15 @@ public class FSTEntry {
private long fileSize = 0;
private long fileOffset = 0;
private Optional<Content> content = Optional.empty();
private boolean isDir = false;
private boolean isRoot = false;
private boolean notInPackage = false;
private short contentFSTID = 0;
private short contentIndex = 0;
}
public void addChildren(FSTEntry entry) {
this.getChildren().add(entry);
}
}

88
src/de/mas/wiiu/jnus/entities/fst/FSTService.java
View File

@ -22,111 +22,67 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import de.mas.wiiu.jnus.entities.content.Content;
import de.mas.wiiu.jnus.entities.content.ContentFSTInfo;
import de.mas.wiiu.jnus.entities.fst.FSTEntry.FSTEntryParam;
import de.mas.wiiu.jnus.utils.ByteUtils;
import lombok.extern.java.Log;
@Log
public final class FSTService {
private FSTService() {
}
public static void parseFST(FSTEntry rootEntry, byte[] fstSection, byte[] namesSection, Map<Integer, Content> contentsByIndex,
Map<Integer, ContentFSTInfo> contentsFSTByIndex, int sectorSize) throws ParseException {
public static void parseFST(FSTEntry rootEntry, byte[] fstSection, byte[] namesSection, int sectorSize) throws ParseException {
int totalEntries = ByteUtils.getIntFromBytes(fstSection, 0x08);
int level = 0;
int[] LEntry = new int[16];
int[] Entry = new int[16];
final HashMap<Integer, FSTEntry> fstEntryToOffsetMap = new HashMap<>();
Entry[level] = 0;
LEntry[level++] = 0;
fstEntryToOffsetMap.put(0, rootEntry);
int lastlevel = level;
final Map<Integer, byte[]> data = new HashMap<>();
for (int i = 1; i < totalEntries; i++) {
data.put(i, Arrays.copyOfRange(fstSection, i * 0x10, (i + 1) * 0x10));
}
int entryOffset = i;
if (level > 0) {
while (LEntry[level - 1] == i) {
level--;
}
}
parseData(1, totalEntries, rootEntry, data, namesSection, sectorSize);
byte[] curEntry = Arrays.copyOfRange(fstSection, i * 0x10, (i + 1) * 0x10);
}
private static int parseData(int i, int end, FSTEntry parent, Map<Integer, byte[]> data, byte[] namesSection, int sectorSize) throws ParseException {
while (i < end) {
byte[] curEntry = data.get(i);
FSTEntryParam entryParam = new FSTEntry.FSTEntryParam();
if (lastlevel != level) {
lastlevel = level;
}
entryParam.setParent(Optional.of(parent));
long fileOffset = ByteUtils.getIntFromBytes(curEntry, 0x04);
long fileSize = ByteUtils.getUnsingedIntFromBytes(curEntry, 0x08);
short flags = ByteUtils.getShortFromBytes(curEntry, 0x0C);
short contentIndex = ByteUtils.getShortFromBytes(curEntry, 0x0E);
if ((curEntry[0] & FSTEntry.FSTEntry_notInNUS) == FSTEntry.FSTEntry_notInNUS) {
entryParam.setNotInPackage(true);
}
FSTEntry parent = null;
if ((curEntry[0] & FSTEntry.FSTEntry_DIR) == FSTEntry.FSTEntry_DIR) {
entryParam.setDir(true);
int parentOffset = (int) fileOffset;
int nextOffset = (int) fileSize;
parent = fstEntryToOffsetMap.get(parentOffset);
Entry[level] = i;
LEntry[level++] = nextOffset;
if (level > 15) {
log.warning("level > 15");
break;
}
} else {
entryParam.setFileOffset(fileOffset * sectorSize);
entryParam.setFileSize(fileSize);
parent = fstEntryToOffsetMap.get(Entry[level - 1]);
}
entryParam.setFlags(flags);
entryParam.setContentIndex(contentIndex);
final int nameOffset = getNameOffset(curEntry);
entryParam.setFileNameSupplier(() -> getName(nameOffset, namesSection));
if (contentsByIndex != null) {
Content content = contentsByIndex.get((int) contentIndex);
if (content == null) {
if ((!entryParam.isDir() || entryParam.isNotInPackage()) && !contentsByIndex.isEmpty()) {
// This is only a problem when the data is NOT on aWUDDataPartition (they have no content files)
log.warning("Content for FST Entry not found");
throw new ParseException("Content for FST Entry not found", 0);
}
} else {
entryParam.setContent(Optional.of(content));
ContentFSTInfo contentFSTInfo = contentsFSTByIndex.get((int) contentIndex);
if (contentFSTInfo == null) {
log.warning("ContentFSTInfo for FST Entry not found");
throw new ParseException("ContentFSTInfo for FST Entry not found", 0);
} else {
content.setContentFSTInfo(contentFSTInfo);
}
}
}
entryParam.setContentFSTID(contentIndex);
entryParam.setParent(Optional.of(parent));
FSTEntry entry = new FSTEntry(entryParam);
fstEntryToOffsetMap.put(entryOffset, entry);
parent.addChildren(entry);
if (entryParam.isDir()) {
i = parseData(i + 1, (int) fileSize, entry, data, namesSection, sectorSize);
} else {
i++;
}
}
return i;
}

13
src/de/mas/wiiu/jnus/implementations/FSTDataProviderNUSTitle.java
View File

@ -76,14 +76,10 @@ public class FSTDataProviderNUSTitle implements FSTDataProvider, HasNUSTitle {
@Override
public boolean readFileToStream(OutputStream out, FSTEntry entry, long offset, Optional<Long> size) throws IOException {
if (!entry.getContent().isPresent()) {
out.close();
throw new IOException("Content for the FSTEntry not found: " + entry);
}
long fileOffset = entry.getFileOffset() + offset;
long fileOffsetBlock = fileOffset;
long usedSize = size.orElse(entry.getFileSize());
Content c = entry.getContent().get();
Content c = title.getTMD().getContentByIndex(entry.getContentIndex());
if (c.isHashed()) {
fileOffsetBlock = (fileOffset / 0xFC00) * 0x10000;
@ -103,19 +99,17 @@ public class FSTDataProviderNUSTitle implements FSTDataProvider, HasNUSTitle {
private boolean decryptFSTEntryToStream(FSTEntry entry, OutputStream outputStream, long fileOffsetBlock, long fileOffset, long fileSize)
throws IOException, CheckSumWrongException, NoSuchAlgorithmException {
if (entry.isNotInPackage() || !entry.getContent().isPresent() || !title.getTicket().isPresent()) {
if (entry.isNotInPackage() || !title.getTicket().isPresent()) {
if (!title.getTicket().isPresent()) {
log.info("Decryption not possible because no ticket was set.");
} else if (entry.isNotInPackage()) {
log.info("Decryption not possible because the FSTEntry is not in this package");
} else if (!entry.getContent().isPresent()) {
log.info("Decryption not possible because the Content was empty");
}
outputStream.close();
return false;
}
Content c = entry.getContent().get();
Content c = title.getTMD().getContentByIndex(entry.getContentIndex());
NUSDataProvider dataProvider = title.getDataProvider();
@ -155,7 +149,6 @@ public class FSTDataProviderNUSTitle implements FSTDataProvider, HasNUSTitle {
sb.append("Hash doesn't match").append(System.lineSeparator());
sb.append("Detailed info:").append(System.lineSeparator());
sb.append(entry).append(System.lineSeparator());
sb.append(entry.getContent()).append(System.lineSeparator());
sb.append(String.format("%016x", title.getTMD().getTitleID()));
sb.append(e.getMessage() + " Calculated Hash: " + Utils.ByteArrayToString(e.getGivenHash()) + ", expected hash: "
+ Utils.ByteArrayToString(e.getExpectedHash()));

14
src/de/mas/wiiu/jnus/implementations/FSTDataProviderWUDDataPartition.java
View File

@ -26,6 +26,7 @@ import de.mas.wiiu.jnus.entities.fst.FSTEntry;
import de.mas.wiiu.jnus.implementations.wud.parser.WUDDataPartition;
import de.mas.wiiu.jnus.implementations.wud.reader.WUDDiscReader;
import de.mas.wiiu.jnus.interfaces.FSTDataProvider;
import de.mas.wiiu.jnus.utils.FSTUtils;
public class FSTDataProviderWUDDataPartition implements FSTDataProvider {
private final WUDDataPartition partition;
@ -50,7 +51,8 @@ public class FSTDataProviderWUDDataPartition implements FSTDataProvider {
@Override
public byte[] readFile(FSTEntry entry, long offset, long size) throws IOException {
ContentFSTInfo info = partition.getFST().getContentFSTInfos().get((int) entry.getContentFSTID());
ContentFSTInfo info = FSTUtils.getFSTInfoForContent(partition.getFST(), entry.getContentIndex())
.orElseThrow(() -> new IOException("Failed to find FSTInfo"));
return getChunkOfData(info.getOffset(), entry.getFileOffset() + offset, size, discReader, titleKey);
}
@ -63,19 +65,21 @@ public class FSTDataProviderWUDDataPartition implements FSTDataProvider {
@Override
public boolean readFileToStream(OutputStream out, FSTEntry entry, long offset, Optional<Long> size) throws IOException {
ContentFSTInfo info = partition.getFST().getContentFSTInfos().get((int) entry.getContentFSTID());
ContentFSTInfo info = FSTUtils.getFSTInfoForContent(partition.getFST(), entry.getContentIndex())
.orElseThrow(() -> new IOException("Failed to find FSTInfo"));
long usedSize = size.orElse(entry.getFileSize());
if (titleKey == null) {
return discReader.readEncryptedToStream(out, partition.getPartitionOffset() + info.getOffset() + entry.getFileOffset() + offset, usedSize);
}
return discReader.readDecryptedToOutputStream(out, partition.getPartitionOffset() + info.getOffset(), entry.getFileOffset() + offset, usedSize, titleKey, null,
false);
return discReader.readDecryptedToOutputStream(out, partition.getPartitionOffset() + info.getOffset(), entry.getFileOffset() + offset, usedSize,
titleKey, null, false);
}
@Override
public InputStream readFileAsStream(FSTEntry entry, long offset, Optional<Long> size) throws IOException {
if (titleKey == null) {
ContentFSTInfo info = partition.getFST().getContentFSTInfos().get((int) entry.getContentFSTID());
ContentFSTInfo info = FSTUtils.getFSTInfoForContent(partition.getFST(), entry.getContentIndex())
.orElseThrow(() -> new IOException("Failed to find FSTInfo"));
long usedSize = size.orElse(entry.getFileSize());
return discReader.readEncryptedToStream(partition.getPartitionOffset() + info.getOffset() + entry.getFileOffset() + offset, usedSize);
}

14
src/de/mas/wiiu/jnus/implementations/NUSDataProviderWUD.java
View File

@ -21,10 +21,13 @@ import java.io.InputStream;
import java.util.Optional;
import de.mas.wiiu.jnus.entities.content.Content;
import de.mas.wiiu.jnus.entities.content.ContentFSTInfo;
import de.mas.wiiu.jnus.entities.fst.FST;
import de.mas.wiiu.jnus.implementations.wud.parser.WUDGamePartition;
import de.mas.wiiu.jnus.implementations.wud.parser.WUDPartitionHeader;
import de.mas.wiiu.jnus.implementations.wud.reader.WUDDiscReader;
import de.mas.wiiu.jnus.interfaces.NUSDataProvider;
import de.mas.wiiu.jnus.utils.FSTUtils;
import lombok.Getter;
import lombok.extern.java.Log;
@ -32,17 +35,24 @@ import lombok.extern.java.Log;
public class NUSDataProviderWUD implements NUSDataProvider {
@Getter private final WUDGamePartition gamePartition;
@Getter private final WUDDiscReader discReader;
@Getter private FST fst;
public NUSDataProviderWUD(WUDGamePartition gamePartition, WUDDiscReader discReader) {
this.gamePartition = gamePartition;
this.discReader = discReader;
}
public long getOffsetInWUD(Content content) {
@Override
public void setFST(FST fst) {
this.fst = fst;
}
public long getOffsetInWUD(Content content) throws IOException {
if (content.getIndex() == 0) { // Index 0 is the FST which is at the beginning of the partion;
return getGamePartition().getPartitionOffset();
}
return getGamePartition().getPartitionOffset() + content.getContentFSTInfo().getOffset();
ContentFSTInfo info = FSTUtils.getFSTInfoForContent(fst, content.getIndex()).orElseThrow(() -> new IOException("Failed to find FSTInfo"));
return getGamePartition().getPartitionOffset() + info.getOffset();
}
@Override

7
src/de/mas/wiiu/jnus/implementations/wud/parser/WUDInfoParser.java
View File

@ -16,7 +16,6 @@
****************************************************************************/
package de.mas.wiiu.jnus.implementations.wud.parser;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.ByteBuffer;
@ -130,7 +129,7 @@ public final class WUDInfoParser {
throw new ParseException("Failed to decrypt the FST of the SI partition.", 0);
}
FST siFST = FST.parseFST(fileTableBlock, new HashMap<>());
FST siFST = FST.parseFST(fileTableBlock);
for (val dirChilden : siFST.getRoot().getDirChildren()) {
// The SI partition contains the tmd, cert and tik for every GM partition.
@ -188,7 +187,7 @@ public final class WUDInfoParser {
throw new IOException("FST Decrpytion failed");
}
FST curFST = FST.parseFST(curFileTableBlock, new HashMap<>());
FST curFST = FST.parseFST(curFileTableBlock);
WUDDataPartition curDataPartition = new WUDDataPartition(curPartionName, partitionOffset + headerSize, curFST);
@ -214,7 +213,7 @@ public final class WUDInfoParser {
throws IOException {
FSTEntry entry = FSTUtils.getEntryByFullPath(fst.getRoot(), filePath).orElseThrow(() -> new FileNotFoundException(filePath + " was not found."));
ContentFSTInfo info = fst.getContentFSTInfos().get((int) entry.getContentFSTID());
ContentFSTInfo info = fst.getContentFSTInfos().get((int) entry.getContentIndex());
if (key == null) {
return discReader.readEncryptedToByteArray(headerSize + partitionOffset + (long) info.getOffset(), entry.getFileOffset(),

5
src/de/mas/wiiu/jnus/interfaces/NUSDataProvider.java
View File

@ -22,6 +22,7 @@ import java.io.InputStream;
import java.util.Optional;
import de.mas.wiiu.jnus.entities.content.Content;
import de.mas.wiiu.jnus.entities.fst.FST;
import de.mas.wiiu.jnus.utils.StreamUtils;
public interface NUSDataProvider {
@ -53,4 +54,8 @@ public interface NUSDataProvider {
public void cleanup() throws IOException;
default public void setFST(FST fst) {
}
}