Implement packing (WIP) #1

This commit is contained in:
Andreas Bielawski 2022-07-03 17:43:56 +02:00
parent 599ff6d41d
commit 4c2287cc65
Signed by: Brawl
GPG Key ID: 851D5FF3B79056CA
11 changed files with 549 additions and 100 deletions

View File

@ -17,8 +17,8 @@ USAGE:
COMMANDS:
unpack, u Unpacks files from a STAR file
pack, p Pack a folder into a STAR file
info, i Shows information about a STAR file
help, h Shows a list of commands or help for one command
GLOBAL OPTIONS:
--help, -h show help (default: false)
@ -26,7 +26,7 @@ GLOBAL OPTIONS:
--version, -v print the version (default: false)
```
### Unpack
### Unpacking
```txt
NAME:
@ -43,6 +43,24 @@ OPTIONS:
If no output directory is given, the file is extracted to the file name minus the extension plus "`_extracted`" (
e.g. `xPackmanJr_0.105.star` -> `xPackmanJr_0.105_extracted`). Same goes for packing (it will append `_packed.star`).
### Packing
**NOTE:** The correct order of the files is not implemented yet and there are many unknowns! See [issue #1](https://github.com/Brawl345/stargazer/issues/1).
```txt
NAME:
stargazer pack - Pack a folder into a STAR file
USAGE:
stargazer pack [command options] [arguments...]
OPTIONS:
--input value, -i value Path to a folder
--output value, -o value Output path of the STAR file. Defaults to '<input folder>_packed.star'
```
If no output STAR file is given, the file will be created in the same directory as the stargazer binary with the name of the folder plus `_packed.star`.
### Info
```txt

View File

@ -1,6 +1,7 @@
package main
import (
"bufio"
"fmt"
"log"
"os"
@ -22,6 +23,7 @@ func main() {
Version: "2.0.0",
Suggest: true,
EnableBashCompletion: true,
HideHelpCommand: true,
Authors: []*cli.Author{
{
Name: "Brawl345",
@ -58,6 +60,28 @@ func main() {
},
Action: unpack,
},
{
Name: "pack",
Aliases: []string{"p"},
Usage: "Pack a folder into a STAR file",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "input",
Aliases: []string{"i"},
Required: true,
Usage: "Path to a folder",
Destination: &input,
},
&cli.StringFlag{
Name: "output",
Aliases: []string{"o"},
Required: false,
Usage: "Output path of the STAR file. Defaults to '<input folder>_packed.star'",
Destination: &output,
},
},
Action: pack,
},
{
Name: "info",
Aliases: []string{"i"},
@ -110,6 +134,49 @@ func unpack(_ *cli.Context) error {
return nil
}
func pack(_ *cli.Context) error {
if output == "" {
output = fmt.Sprintf("%s_packed.star", filepath.Base(input))
}
if !quiet {
log.Printf("Will pack to '%s'", output)
}
if !quiet {
log.Printf("Reading '%s'...", input)
}
star, err := stargazer.NewSTARFileFromDirectory(input)
if err != nil {
return err
}
if !quiet {
log.Printf("Writing to '%s'...\n", output)
}
out, err := os.Create(output)
if err != nil {
return err
}
defer out.Close()
writer := bufio.NewWriter(out)
_, err = star.WriteTo(writer)
if err != nil {
return err
}
err = writer.Flush()
if err != nil {
return err
}
return nil
}
func info(_ *cli.Context) error {
star, err := stargazer.LoadSTARFromFile(input)
if err != nil {

7
pkg/stargazer/const.go Normal file
View File

@ -0,0 +1,7 @@
package stargazer
import "math"
const (
MaxFilenameSize = math.MaxUint8 - 8
)

View File

@ -3,6 +3,7 @@ package stargazer
import (
"errors"
"fmt"
"math"
)
var ErrNoEntries = errors.New("invalid STAR file - no file entries found")
@ -13,8 +14,40 @@ type (
Actual string
Filename string
}
ErrFilenameTooLong struct {
Filename string
}
ErrFileTooLarge struct {
Filename string
}
ErrNotAFile struct {
Filename string
}
ErrNotADirectory struct {
Path string
}
)
func (e ErrSHA1Mismatch) Error() string {
return fmt.Sprintf("SHA1 mismatch on file '%s': expected %s, actual %s", e.Filename, e.Expected, e.Actual)
}
func (e ErrFilenameTooLong) Error() string {
return fmt.Sprintf("filename '%s' is too long, needs to be < %d characters.", e.Filename, MaxFilenameSize)
}
func (e ErrFileTooLarge) Error() string {
return fmt.Sprintf("file '%s' is too large, needs to be < %d bytes.", e.Filename, math.MaxUint32)
}
func (e ErrNotAFile) Error() string {
return fmt.Sprintf("'%s' is not a file.", e.Filename)
}
func (e ErrNotADirectory) Error() string {
return fmt.Sprintf("'%s' is not a directory.", e.Path)
}

View File

@ -1,88 +0,0 @@
package stargazer
import (
"fmt"
"os"
"path/filepath"
"strings"
)
type (
Star struct {
Entries []Entry
}
Entry struct {
Header
Content []byte
SHA1 [20]byte
}
Header struct {
Headersize uint8
Padding1 uint8
Filesize uint32
FilenameSize uint8
Padding2 uint8
Filename string
}
)
func (e *Entry) SHA1String() string {
return fmt.Sprintf("%x", e.SHA1)
}
func (e *Entry) Unpack(outputDir string) error {
fp := filepath.Join(outputDir, e.Filename)
err := os.MkdirAll(filepath.Dir(fp), os.ModePerm)
if err != nil {
return err
}
f, err := os.Create(fp)
if err != nil {
return err
}
defer f.Close()
_, err = f.Write(e.Content)
if err != nil {
return err
}
return nil
}
func (e *Entry) Info() string {
var sb strings.Builder
sb.WriteString(fmt.Sprintf("%s\n", e.Filename))
sb.WriteString(fmt.Sprintf(" Filesize: %d bytes\n", e.Header.Filesize))
sb.WriteString(fmt.Sprintf(" SHA1: %s\n", e.SHA1String()))
return sb.String()
}
func (s *Star) Unpack(outputDir string) error {
for _, e := range s.Entries {
err := e.Unpack(outputDir)
if err != nil {
return err
}
}
return nil
}
func (s *Star) Info() string {
var sb strings.Builder
var contentSize uint64
for _, e := range s.Entries {
contentSize += uint64(e.Header.Filesize)
sb.WriteString(e.Info())
sb.WriteString("\n")
}
sb.WriteString(fmt.Sprintf("Total contents: %d\n", len(s.Entries)))
sb.WriteString(fmt.Sprintf("Total content size: %d bytes\n", contentSize))
return sb.String()
}

View File

@ -7,7 +7,31 @@ import (
"encoding/hex"
"fmt"
"io"
"math"
"os"
"path/filepath"
"strings"
)
type (
Star struct {
Entries []Entry
}
Entry struct {
Header
Filename string
Content []byte
SHA1 [20]byte
}
Header struct {
Headersize uint8
Padding1 uint8
Filesize uint32
FilenameSize uint8
Padding2 uint8
}
)
func parseEntry(file io.Reader) (*Entry, error) {
@ -60,7 +84,7 @@ func parseEntry(file io.Reader) (*Entry, error) {
}
return nil, fmt.Errorf("error reading filename: %v", err)
}
entry.Header.Filename = string(filename)
entry.Filename = string(filename)
entry.Content = make([]byte, entry.Header.Filesize)
err = binary.Read(file, binary.LittleEndian, &entry.Content)
@ -85,7 +109,7 @@ func parseEntry(file io.Reader) (*Entry, error) {
if !bytes.Equal(calculatedHash, entry.SHA1[:]) {
return nil, ErrSHA1Mismatch{
Filename: entry.Header.Filename,
Filename: entry.Filename,
Expected: hex.EncodeToString(entry.SHA1[:]),
Actual: hex.EncodeToString(calculatedHash),
}
@ -94,15 +118,109 @@ func parseEntry(file io.Reader) (*Entry, error) {
return &entry, nil
}
func LoadSTARFromFile(fp string) (*Star, error) {
//NewSTARFileFromDirectory creates a new STAR from a given directory.
func NewSTARFileFromDirectory(dir string) (*Star, error) {
// TODO: Order files with install.txt and metadata.txt at end
if !isDir(dir) {
return nil, ErrNotADirectory{
Path: dir,
}
}
star := &Star{}
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
relativePath, err := filepath.Rel(dir, path)
if err != nil {
return err
}
entry, err := NewEntryFromFile(dir, relativePath)
if err != nil {
return err
}
star.Entries = append(star.Entries, *entry)
return nil
})
if err != nil {
return nil, err
}
return star, nil
}
//NewEntryFromFile creates a new STAR entry from a file.
//Second parameter is the relative filename of the file in the archive.
func NewEntryFromFile(dir string, f string) (*Entry, error) {
fp := filepath.Join(dir, f)
file, err := os.Open(fp)
if err != nil {
return nil, err
}
defer file.Close()
entry := &Entry{}
filename := strings.TrimPrefix(f, string(os.PathSeparator))
filename = strings.ReplaceAll(filename, "\\", "/")
if len(filename) > MaxFilenameSize {
return nil, ErrFilenameTooLong{
Filename: filename,
}
}
entry.Filename = filename
entry.Header.FilenameSize = uint8(len(entry.Filename))
entry.Content, err = io.ReadAll(file)
if err != nil {
return nil, err
}
if len(entry.Content) > math.MaxUint32 {
return nil, ErrFileTooLarge{
Filename: filename,
}
}
entry.Header.Filesize = uint32(len(entry.Content))
entry.Header.Headersize = uint8(1 + 1 + 4 + 1 + 1 + len(entry.Filename))
h := sha1.New()
h.Write(entry.Content)
copy(entry.SHA1[:], h.Sum(nil))
return entry, nil
}
//LoadSTARFromFile loads a STAR file from a filepath.
func LoadSTARFromFile(fp string) (*Star, error) {
file, err := os.Open(fp)
if err != nil {
return nil, err
}
defer file.Close()
if !isFile(fp) {
return nil, ErrNotAFile{
Filename: fp,
}
}
return LoadSTAR(file)
}
//LoadSTAR loads a STAR file from an io.Reader.
func LoadSTAR(file io.Reader) (*Star, error) {
star := &Star{}
for {
@ -122,3 +240,110 @@ func LoadSTAR(file io.Reader) (*Star, error) {
return star, nil
}
//SHA1String returns the SHA1 of the file as a hex string.
func (e *Entry) SHA1String() string {
return fmt.Sprintf("%x", e.SHA1)
}
//Unpack unpacks the entry to the given directory.
func (e *Entry) Unpack(outputDir string) error {
fp := filepath.Join(outputDir, e.Filename)
err := os.MkdirAll(filepath.Dir(fp), os.ModePerm)
if err != nil {
return err
}
f, err := os.Create(fp)
if err != nil {
return err
}
defer f.Close()
_, err = f.Write(e.Content)
if err != nil {
return err
}
return nil
}
//WriteTo writes the entry to the given io.Writer.
func (e *Entry) WriteTo(w io.Writer) (int64, error) {
var total int64
err := binary.Write(w, binary.LittleEndian, &e.Header)
if err != nil {
return 0, err
}
total += int64(e.Header.Headersize)
err = binary.Write(w, binary.LittleEndian, []byte(e.Filename))
if err != nil {
return 0, err
}
total += int64(e.Header.FilenameSize)
err = binary.Write(w, binary.LittleEndian, e.Content)
if err != nil {
return 0, err
}
total += int64(e.Header.Filesize)
err = binary.Write(w, binary.LittleEndian, e.SHA1)
if err != nil {
return 0, err
}
total += int64(20)
return total, nil
}
//Info returns a string with various information of the entry.
func (e *Entry) Info() string {
var sb strings.Builder
sb.WriteString(fmt.Sprintf("%s\n", e.Filename))
sb.WriteString(fmt.Sprintf(" Filesize: %d bytes\n", e.Header.Filesize))
sb.WriteString(fmt.Sprintf(" SHA1: %s\n", e.SHA1String()))
return sb.String()
}
//Unpack unpacks all entries from the STAR to the given directory.
func (s *Star) Unpack(outputDir string) error {
for _, e := range s.Entries {
err := e.Unpack(outputDir)
if err != nil {
return err
}
}
return nil
}
//WriteTo writes the STAR with all entries to the given io.Writer.
func (s *Star) WriteTo(w io.Writer) (int64, error) {
var total int64
for _, e := range s.Entries {
n, err := e.WriteTo(w)
if err != nil {
return total, err
}
total += n
}
return total, nil
}
//Info returns a string with various information of all entries.
func (s *Star) Info() string {
var sb strings.Builder
var contentSize uint64
for _, e := range s.Entries {
contentSize += uint64(e.Header.Filesize)
sb.WriteString(e.Info())
sb.WriteString("\n")
}
sb.WriteString(fmt.Sprintf("Total contents: %d\n", len(s.Entries)))
sb.WriteString(fmt.Sprintf("Total content size: %d bytes\n", contentSize))
return sb.String()
}

View File

@ -1,6 +1,7 @@
package stargazer
import (
"errors"
"os"
"path/filepath"
"testing"
@ -44,8 +45,8 @@ func TestLoadSTARFromFile(t *testing.T) {
return
}
if got.Entries[0].Header.Filename != "NulledFile.rel" {
t.Errorf("Expected filename of first entry to be 'NulledFile.rel', got '%s'", got.Entries[0].Header.Filename)
if got.Entries[0].Filename != "NulledFile.rel" {
t.Errorf("Expected filename of first entry to be 'NulledFile.rel', got '%s'", got.Entries[0].Filename)
return
}
@ -90,8 +91,8 @@ func TestLoadSTARFromFile(t *testing.T) {
return
}
if got.Entries[1].Header.Filename != "metadata.xml" {
t.Errorf("Expected filename of second entry to be 'metadata.xml', got '%s'", got.Entries[1].Header.Filename)
if got.Entries[1].Filename != "metadata.xml" {
t.Errorf("Expected filename of second entry to be 'metadata.xml', got '%s'", got.Entries[1].Filename)
return
}
@ -136,8 +137,8 @@ func TestLoadSTARFromFile(t *testing.T) {
return
}
if got.Entries[2].Header.Filename != "install.txt" {
t.Errorf("Expected filename of third entry to be 'install.txt', got '%s'", got.Entries[2].Header.Filename)
if got.Entries[2].Filename != "install.txt" {
t.Errorf("Expected filename of third entry to be 'install.txt', got '%s'", got.Entries[2].Filename)
return
}
@ -157,7 +158,7 @@ func TestLoadSTARFromFile(t *testing.T) {
}
}
func TestLoadSTARFromFileFail(t *testing.T) {
func TestLoadSTARFromFileNotExisting(t *testing.T) {
_, err := LoadSTARFromFile("invalid.star")
if err == nil {
t.Errorf("Expected error, got nil")
@ -165,6 +166,20 @@ func TestLoadSTARFromFileFail(t *testing.T) {
}
}
func TestLoadSTARFromFileNotAFile(t *testing.T) {
_, err := LoadSTARFromFile(".")
if err == nil {
t.Errorf("Expected error, got nil")
return
}
var expectedErr ErrNotAFile
if !errors.As(err, &expectedErr) {
t.Errorf("Expected error to be of type ErrNotAFile, got %T", err)
return
}
}
func TestStar_Unpack(t *testing.T) {
got, err := LoadSTARFromFile(filepath.Join("..", "..", "testdata", "testfile.star"))
if err != nil {
@ -195,6 +210,136 @@ func TestStar_Unpack(t *testing.T) {
}
}
func TestNewEntryFromFile(t *testing.T) {
got, err := NewEntryFromFile(filepath.Join("..", "..", "testdata", "unpacked"), filepath.Join("dummyfolder", "NulledFile.rel"))
if err != nil {
t.Error(err)
return
}
if got.Filename != "dummyfolder/NulledFile.rel" {
t.Errorf("Expected filename to be 'dummyfolder/NulledFile', got '%s'", got.Filename)
return
}
if got.Header.Headersize != 34 {
t.Errorf("Expected header size to be 34, got %d", got.Header.Headersize)
return
}
if got.Header.Padding1 != 0 {
t.Errorf("Expected first padding to be 0, got %d", got.Header.Padding1)
return
}
if got.Header.Filesize != 10928 {
t.Errorf("Expected filesize to be 10928, got %d", got.Header.Filesize)
return
}
if got.Header.FilenameSize != 26 {
t.Errorf("Expected filename size to be 26, got %d", got.Header.FilenameSize)
return
}
if got.Header.Padding2 != 0 {
t.Errorf("Expected second padding to be 0, got %d", got.Header.Padding2)
return
}
if got.SHA1String() != "3d433fcbe9585b05ea877814bad60774ff8a9e7c" {
t.Errorf("Expected SHA1 to be '3d433fcbe9585b05ea877814bad60774ff8a9e7c', got '%s'", got.SHA1String())
return
}
if got.Content == nil {
t.Errorf("Expected content to be non-nil")
return
}
if uint32(len(got.Content)) != got.Header.Filesize {
t.Errorf("Content size in header is set to %d bytes, but the file is actually %d bytes long", got.Header.Filesize, len(got.Content))
return
}
}
func TestNewSTARFileFromDirectory(t *testing.T) {
got, err := NewSTARFileFromDirectory(filepath.Join("..", "..", "testdata", "unpacked"))
if err != nil {
t.Error(err)
return
}
if len(got.Entries) != 3 {
t.Errorf("Expected 3 entries, got %d", len(got.Entries))
return
}
// TODO: Test entries when the correct order is implemented
// Check first entry
if got.Entries[0].Header.Headersize != 34 {
t.Errorf("Expected header size of first entry to be 34 bytes long, got %d", got.Entries[0].Header.Headersize)
return
}
if got.Entries[0].Header.Padding1 != 0 {
t.Errorf("Expected first padding of first entry to be 0, got %d", got.Entries[0].Header.Padding1)
return
}
if got.Entries[0].Header.Filesize != 10928 {
t.Errorf("Expected filesize of first entry to be 10928, got %d", got.Entries[0].Header.Filesize)
return
}
if got.Entries[0].Header.FilenameSize != 26 {
t.Errorf("Expected filename size of first entry to be 26, got %d", got.Entries[0].Header.FilenameSize)
return
}
if got.Entries[0].Header.Padding2 != 0 {
t.Errorf("Expected second padding of first entry to be 0, got %d", got.Entries[0].Header.Padding2)
return
}
if got.Entries[0].Filename != "dummyfolder/NulledFile.rel" {
t.Errorf("Expected filename of first entry to be 'dummyfolder/NulledFile.rel', got '%s'", got.Entries[0].Filename)
return
}
if got.Entries[0].Content == nil {
t.Errorf("Expected content of first entry to be non-nil")
return
}
if uint32(len(got.Entries[0].Content)) != got.Entries[0].Header.Filesize {
t.Errorf("Expected content of first entry to be %d bytes long, got %d", got.Entries[0].Header.Filesize, len(got.Entries[0].Content))
return
}
if got.Entries[0].SHA1String() != "3d433fcbe9585b05ea877814bad60774ff8a9e7c" {
t.Errorf("Expected SHA1 of first entry to be '3d433fcbe9585b05ea877814bad60774ff8a9e7c', got '%s'", got.Entries[0].SHA1String())
return
}
}
func TestNewSTARFileFromDirectoryNotADirectory(t *testing.T) {
_, err := NewSTARFileFromDirectory(filepath.Join("..", "..", "testdata", "install.txt"))
if err == nil {
t.Errorf("Expected error, got nil")
return
}
var expectedErr ErrNotADirectory
if !errors.As(err, &expectedErr) {
t.Errorf("Expected error to be of type ErrNotADirectory, got %T", err)
return
}
}
// TODO: More tests for failures (file size, file name size, etc.)
func fileExists(fileName string) bool {
if _, err := os.Stat(fileName); err == nil {
return true

19
pkg/stargazer/utils.go Normal file
View File

@ -0,0 +1,19 @@
package stargazer
import "os"
func isFile(path string) bool {
st, err := os.Stat(path)
if err != nil {
return false
}
return st.Mode().IsRegular()
}
func isDir(path string) bool {
st, err := os.Stat(path)
if err != nil {
return false
}
return st.IsDir()
}

Binary file not shown.

9
testdata/unpacked/install.txt vendored Normal file
View File

@ -0,0 +1,9 @@
# --- Uninstall previous version
RemovePackage %%%Name%%% %%%Major%%%
DeleteDirectory %%%PackagesDir%%%/%%%Name%%%_%%%Major%%%
# --- Install new version
CreateDirectory %%%PackagesDir%%%/%%%Name%%%_%%%Major%%%
CopyDirectory %%%DownloadDir%%%/%%%Name%%%_%%%Major%%%.%%%Minor%%% %%%PackagesDir%%%/%%%Name%%%_%%%Major%%%
AddPackage %%%Name%%% %%%Version%%% %%%PackagesDir%%%/%%%Name%%%_%%%Major%%% flash_package

14
testdata/unpacked/metadata.xml vendored Normal file
View File

@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<Package>
<ID>000</ID>
<Name>NulledFile</Name>
<Title> </Title>
<PlatformName>PSXJA3</PlatformName>
<Description> </Description>
<Files/>
<Dependencies/>
<Version>
<Major>0</Major>
<Minor>100</Minor>
</Version>
</Package>