From aff22d28da6f9e6699b63b883dc08cb90bb24e37 Mon Sep 17 00:00:00 2001 From: barubary Date: Sun, 13 Nov 2011 22:21:18 +0000 Subject: [PATCH] C#: revised implementation of the tree-writing algorithm for the 8-bit huffman format. A test with a file containing all values from 0 to 0xFF evenly now does not result in a corrupt tree definition. --- CSharp/DSDecmp/Formats/Nitro/Huffman.cs | 251 ++++++++++++++++++------ CSharp/PluginDistro/GoldenSunDD.dll | Bin 6656 -> 6656 bytes CSharp/PluginDistro/LuminousArc.dll | Bin 6656 -> 6656 bytes 3 files changed, 193 insertions(+), 58 deletions(-) diff --git a/CSharp/DSDecmp/Formats/Nitro/Huffman.cs b/CSharp/DSDecmp/Formats/Nitro/Huffman.cs index 803d215..1632678 100644 --- a/CSharp/DSDecmp/Formats/Nitro/Huffman.cs +++ b/CSharp/DSDecmp/Formats/Nitro/Huffman.cs @@ -222,6 +222,7 @@ namespace DSDecmp.Formats.Nitro /// public class HuffTreeNode { + #region Fields & Properties: Data & IsData /// /// The data contained in this node. May not mean anything when isData == false /// @@ -251,7 +252,9 @@ namespace DSDecmp.Formats.Nitro /// Returns true if this node represents data. /// public bool IsData { get { return this.isData; } } + #endregion + #region Field & Properties: Children & Parent /// /// The child of this node at side 0 /// @@ -280,7 +283,9 @@ namespace DSDecmp.Formats.Nitro /// Determines if this is the Child1 of the parent node. Assumes there is a parent. /// public bool IsChild1 { get { return this.Parent.child1 == this; } } + #endregion + #region Field & Property: Depth private int depth; /// /// Get or set the depth of this node. Will not be set automatically, but @@ -300,7 +305,9 @@ namespace DSDecmp.Formats.Nitro } } } + #endregion + #region Property: Size /// /// Calculates the size of the sub-tree with this node as root. /// @@ -313,25 +320,15 @@ namespace DSDecmp.Formats.Nitro return 1 + this.child0.Size + this.child1.Size; } } + #endregion /// - /// Returns a seuqnce over the nodes of the sub-tree with this node as root in a pre-order fashion. (Root-Left-Right) + /// The index of this node in the array for building the proper ordering. + /// If -1, this node has not yet been placed in the array. /// - public IEnumerable PreOrderTraversal - { - get - { - yield return this; - if (!this.IsData) - { - foreach (HuffTreeNode c in this.child0.PreOrderTraversal) - yield return c; - foreach (HuffTreeNode c in this.child1.PreOrderTraversal) - yield return c; - } - } - } + internal int index = -1; + #region Constructor(data, isData, child0, child1) /// /// Manually creates a new node for a huffman tree. /// @@ -352,7 +349,9 @@ namespace DSDecmp.Formats.Nitro this.child1.Parent = this; } } + #endregion + #region Constructor(Stream, isData, relOffset, maxStreamPos) /// /// Creates a new node in the Huffman tree. /// @@ -414,6 +413,7 @@ namespace DSDecmp.Formats.Nitro stream.Position = currStreamPos; } } + #endregion public override string ToString() { @@ -531,6 +531,7 @@ namespace DSDecmp.Formats.Nitro compressedLength++; // use a breadth-first traversal to store the tree, such that we do not need to store/calculate the side of each sub-tree. + // because the data is only 4 bits long, no tree will ever let the offset field overflow. LinkedList printQueue = new LinkedList(); printQueue.AddLast(root); while (printQueue.Count > 0) @@ -547,6 +548,8 @@ namespace DSDecmp.Formats.Nitro // bit 6: node1 end flag // bit 7: node0 end flag byte data = (byte)(printQueue.Count / 2); + if (data > 0x3F) + throw new InvalidDataException("BUG: offset overflow in 4-bit huffman."); data = (byte)(data & 0x3F); if (node.Child0.IsData) data |= 0x80; @@ -712,59 +715,99 @@ namespace DSDecmp.Formats.Nitro compressedLength++; // use a breadth-first traversal to store the tree, such that we do not need to store/calculate the size of each sub-tree. - // NO! BF results in an ordering that may overflow the offset field. use pre-order instead (Self-Left-Right) - foreach (HuffTreeNode node in root.PreOrderTraversal) + // NO! BF results in an ordering that may overflow the offset field. + + // find the BF order of all nodes that have two leaves as children. We're going to insert them in an array in reverse BF order, + // inserting the parent whenever both children have been inserted. + + LinkedList leafStemQueue = new LinkedList(); + + #region fill the leaf queue; first->last will be reverse BF + LinkedList nodeCodeStack = new LinkedList(); + nodeCodeStack.AddLast(root); + while (nodeCodeStack.Count > 0) { - if (node.Parent == null) // root node. - { - // bits 0-5: 'offset' = # nodes in queue left - // bit 6: node1 end flag - // bit 7: node0 end flag - byte data = 0; - if (node.Child0.IsData) - data |= 0x80; - if (node.Child1.IsData) - data |= 0x40; - outstream.WriteByte(data); - } + HuffTreeNode node = nodeCodeStack.First.Value; + nodeCodeStack.RemoveFirst(); if (node.IsData) continue; + if (node.Child0.IsData && node.Child1.IsData) + { + leafStemQueue.AddFirst(node); + } else { - // bits 0-5: 'offset': if this is left node, 0. if right node - // bit 6: node1 end flag - // bit 7: node0 end flag + nodeCodeStack.AddLast(node.Child0); + nodeCodeStack.AddLast(node.Child1); } - compressedLength++; + } - /* - while (printQueue.Count > 0) + #endregion + + HuffTreeNode[] nodeArray = new HuffTreeNode[0x1FF]; // this array does not contain the leaves themselves! + while (leafStemQueue.Count > 0) { - HuffTreeNode node = printQueue.First.Value; - printQueue.RemoveFirst(); - if (node.IsData) - { - outstream.WriteByte(node.Data); - } - else - { - // bits 0-5: 'offset' = # nodes in queue left - // bit 6: node1 end flag - // bit 7: node0 end flag - byte data = (byte)(printQueue.Count / 2); - data = (byte)(data & 0x3F); - if (node.Child0.IsData) - data |= 0x80; - if (node.Child1.IsData) - data |= 0x40; - outstream.WriteByte(data); + Insert(leafStemQueue.First.Value, nodeArray, 0x3F + 1); + leafStemQueue.RemoveFirst(); + } - printQueue.AddFirst(node.Child1); - printQueue.AddFirst(node.Child0); - } - compressedLength++; - }/**/ + // update the indices to ignore all gaps + int nodeIndex = 0; + for (int i = 0; i < nodeArray.Length; i++) + { + if (nodeArray[i] != null) + nodeArray[i].index = nodeIndex++; + } + // write the nodes in their given order. However when 'writing' a node, write the data of its children instead. + // the root node is always the first node. + byte rootData = 0; + if (root.Child0.IsData) + rootData |= 0x80; + if (root.Child1.IsData) + rootData |= 0x40; + outstream.WriteByte(rootData); compressedLength++; + + for (int i = 0; i < nodeArray.Length; i++) + { + if (nodeArray[i] != null) + { + // nodes in this array are never data! + HuffTreeNode node0 = nodeArray[i].Child0; + if (node0.IsData) + outstream.WriteByte(node0.Data); + else + { + int offset = node0.index - nodeArray[i].index - 1; + if (offset > 0x3F) + throw new Exception("Offset overflow!"); + byte data = (byte)offset; + if (node0.Child0.IsData) + data |= 0x80; + if (node0.Child1.IsData) + data |= 0x40; + outstream.WriteByte(data); + } + + HuffTreeNode node1 = nodeArray[i].Child1; + if (node1.IsData) + outstream.WriteByte(node1.Data); + else + { + int offset = node1.index - nodeArray[i].index - 1; + if (offset > 0x3F) + throw new Exception("Offset overflow!"); + byte data = (byte)offset; + if (node0.Child0.IsData) + data |= 0x80; + if (node0.Child1.IsData) + data |= 0x40; + outstream.WriteByte(data); + } + + compressedLength += 2; + } + } #endregion #region write the data @@ -813,6 +856,98 @@ namespace DSDecmp.Formats.Nitro return compressedLength; } #endregion + + #region Utility Method: Insert(node, HuffTreeNode[], maxOffset) + /// + /// Inserts the given node into the given array, in such a location that + /// the offset to both of its children is at most the given maximum, and as large as possible. + /// In order to do this, the contents of the array may be shifted to the right. + /// + /// The node to insert. + /// The array to insert the node in. + /// The maximum offset between parent and children. + private void Insert(HuffTreeNode node, HuffTreeNode[] array, int maxOffset) + { + // if the node has two data-children, insert it as far to the end as possible. + if (node.Child0.IsData && node.Child1.IsData) + { + for (int i = array.Length - 1; i >= 0; i--) + { + if (array[i] == null) + { + array[i] = node; + node.index = i; + break; + } + } + } + else + { + // if the node is not data, insert it as far left as possible. + // we know that both children are already present. + int offset = Math.Max(node.Child0.index - maxOffset, node.Child1.index - maxOffset); + offset = Math.Max(0, offset); + if (offset >= node.Child0.index || offset >= node.Child1.index) + { + // it may be that the childen are too far apart, with lots of empty entries in-between. + // shift the bottom child right until the node fits in its left-most place for the top child. + // (there should be more than enough room in the array) + while (offset >= Math.Min(node.Child0.index, node.Child1.index)) + ShiftRight(array, Math.Min(node.Child0.index, node.Child1.index), maxOffset); + while (array[offset] != null) + ShiftRight(array, offset, maxOffset); + array[offset] = node; + node.index = offset; + } + else + { + for (int i = offset; i < node.Child0.index && i < node.Child1.index; i++) + { + if (array[i] == null) + { + array[i] = node; + node.index = i; + break; + } + } + } + } + + if (node.index < 0) + throw new Exception("Node could not be inserted!"); + + // if the insertion of this node means that the parent has both children inserted, insert the parent. + if (node.Parent != null) + { + if ((node.Parent.Child0.index >= 0 || node.Parent.Child0.IsData) + && (node.Parent.Child1.index >= 0 || node.Parent.Child1.IsData)) + Insert(node.Parent, array, maxOffset); + } + } + #endregion + + #region Utility Method: ShiftRight(HuffTreeNode[], index, maxOffset) + /// + /// Shifts the node at the given index one to the right. + /// If the distance between parent and child becomes too large due to this shift, the parent is shifted as well. + /// + /// The array to shift the node in. + /// The index of the node to shift. + /// The maximum distance between parent and children. + private void ShiftRight(HuffTreeNode[] array, int idx, int maxOffset) + { + HuffTreeNode node = array[idx]; + if (array[idx + 1] != null) + ShiftRight(array, idx + 1, maxOffset); + if (node.Parent.index > 0 && node.index - maxOffset + 1 > node.Parent.index) + ShiftRight(array, node.Parent.index, maxOffset); + if (node != array[idx]) + return; // already done indirectly. + array[idx + 1] = array[idx]; + array[idx] = null; + node.index++; + } + #endregion } public class HuffmanAny : CompositeFormat diff --git a/CSharp/PluginDistro/GoldenSunDD.dll b/CSharp/PluginDistro/GoldenSunDD.dll index 966b963508a940c4d8c4e3b6466fe187070d25c1..9274000cfe5fef8603825bd152a9a302546492e9 100644 GIT binary patch delta 49 zcmZoLX)u}4!6fm2W7jEufrWf-??qOcdNhYT{O7evR(ErXfDa>3a`GlYEk>cucLkL> E0QH~}!~g&Q delta 49 zcmZoLX)u}4!BqKbW7jEufli(Vg*LYa@5P)e`b*L$7;jDy@L>c>PTnM_#VEM>uAnjp E0PZmojsO4v diff --git a/CSharp/PluginDistro/LuminousArc.dll b/CSharp/PluginDistro/LuminousArc.dll index 75ff5f026d6ea47c358aba384e5478cb30920b6d..2540b907fc55c657437b35d0902d1af03d8778c5 100644 GIT binary patch delta 49 zcmZoLX)u}4!6fm2W7j(Yfek&mRy);gU6y^gH7&_Bu6}ceU<@Nra`H1FEk@zZg2M0F E01@yL_y7O^ delta 49 zcmZoLX)u}4!BqKbW7j(YfzYo#IaxWa-gEuu8ZSF?CwFs)U<@Nra`H1FEk>cug2M0F E05D?}C;$Ke