From 6ba534166b538d8efc42561bd63acbe2d18797f8 Mon Sep 17 00:00:00 2001 From: Mateusz Faderewski Date: Sat, 5 Aug 2023 01:37:09 +0200 Subject: [PATCH] Simplified build action + ROM stripping fix + Makefile fix --- .github/workflows/build.yml | 109 +++++++++++++----------------------- Doxyfile | 4 +- Makefile | 52 ++++++++++++----- README.md | 3 +- localdeploy.bat | 9 ++- remotedeploy.sh | 6 +- src/flashcart/flashcart.c | 2 +- tools/README.md | 9 +++ tools/sc64/.gitkeep | 0 tools/sc64/README.md | 12 ---- tools/sc64/minify.py | 39 ------------- tools/strip_debug_data.py | 58 +++++++++++++++++++ 12 files changed, 158 insertions(+), 145 deletions(-) mode change 100644 => 100755 remotedeploy.sh create mode 100644 tools/README.md create mode 100644 tools/sc64/.gitkeep delete mode 100644 tools/sc64/README.md delete mode 100644 tools/sc64/minify.py create mode 100644 tools/strip_debug_data.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 725cd5f4..5903f43e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,112 +9,81 @@ on: workflow_dispatch: jobs: - build-sc64-menu: + build-menu: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 with: submodules: recursive - fetch-depth: 1 # we only require the last check-in, unless we want to create a changelog. - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Install pinned version of devcontainers cli (https://github.com/devcontainers/ci/issues/252#issuecomment-1654523080) - shell: bash - run: | - npm install --global @devcontainers/cli@0.50.0 - - name: Build N64FlashcartMenu ROM uses: devcontainers/ci@v0.3 with: - push: never - runCmd: | - mkdir build - mkdir output - # TODO: split this to use params for each flashcart type. - make + imageName: ghcr.io/Polprzewodnikowy/N64FlashcartMenu-devcontainer + cacheFrom: ghcr.io/Polprzewodnikowy/N64FlashcartMenu-devcontainer + push: always + runCmd: make all + env: + FLAGS: -DMENU_RELEASE - - name: Upload artifact + - name: Upload artifact (Standard ROM) uses: actions/upload-artifact@v3 with: name: N64FlashcartMenu path: | - ./output/N64FlashcartMenu.z64 + ./output/N64FlashcartMenu.n64 ./build/N64FlashcartMenu.elf - minify-sc64-menu: - runs-on: ubuntu-latest - needs: build-sc64-menu - - steps: - - - name: Setup python - uses: actions/setup-python@v4 - with: - python-version: '3.11.x' - - - uses: actions/checkout@v3 - with: - fetch-depth: 1 # we only require the last check-in, unless we want to create a changelog. - - - name: Download ROM artifact - id: download-rom-artifact - uses: actions/download-artifact@v3 - with: - name: N64FlashcartMenu - path: ./ - - - name: Finalize ROM - run: | - # make all - python ./tools/sc64/minify.py ./build/N64FlashcartMenu.elf ./output/N64FlashcartMenu.z64 ./output/sc64menu.n64 - continue-on-error: false - - - name: Upload artifact + - name: Upload artifact (64drive version) uses: actions/upload-artifact@v3 with: - name: SC64-Menu - path: | - ./output/sc64menu.n64 - if-no-files-found: ignore + name: 64drive + path: ./output/menu.bin + + - name: Upload artifact (ED64 version) + uses: actions/upload-artifact@v3 + with: + name: ED64 + path: ./output/OS64.v64 + + - name: Upload artifact (SC64 version) + uses: actions/upload-artifact@v3 + with: + name: SC64 + path: ./output/sc64menu.n64 generate-docs: runs-on: ubuntu-latest - # needs: build-sc64-menu steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - fetch-depth: 1 # we only require the last check-in, unless we want to create a changelog. - name: Run Doxygen uses: mattnotmitt/doxygen-action@1.9.5 with: doxyfile-path: './Doxyfile' - # - name: Deploy - # uses: peaceiris/actions-gh-pages@v3 - # with: - # github_token: ${{ secrets.GITHUB_TOKEN }} - # publish_dir: ./docs + - name: Deploy to GitHub Pages + uses: peaceiris/actions-gh-pages@v3 + if: github.ref == 'refs/heads/main' + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./output/docs +# release-sc64-menu: +# runs-on: ubuntu-latest +# needs: minify-sc64-menu - # release-sc64-menu: - # runs-on: ubuntu-latest - # needs: minify-sc64-menu - - # steps: - # - name: Generate release - # if: github.event_name == 'release' && github.event.action == 'created' - # run: | - # echo "still release preview. Check actions for build assets." - +# steps: +# - name: Generate release +# if: github.event_name == 'release' && github.event.action == 'created' +# run: | +# echo "still release preview. Check actions for build assets." diff --git a/Doxyfile b/Doxyfile index a61d207e..7b99e3b5 100644 --- a/Doxyfile +++ b/Doxyfile @@ -67,7 +67,7 @@ PROJECT_LOGO = # entered, it will be relative to the location where doxygen was started. If # left blank the current directory will be used. -OUTPUT_DIRECTORY = docs +OUTPUT_DIRECTORY = output # If the CREATE_SUBDIRS tag is set to YES then doxygen will create up to 4096 # sub-directories (in 2 levels) under the output directory of each output format @@ -1220,7 +1220,7 @@ GENERATE_HTML = YES # The default directory is: html. # This tag requires that the tag GENERATE_HTML is set to YES. -HTML_OUTPUT = ref +HTML_OUTPUT = docs # The HTML_FILE_EXTENSION tag can be used to specify the file extension for each # generated HTML page (for example: .htm, .php, .asp). diff --git a/Makefile b/Makefile index f2b81c5d..062d2848 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PROJECT_NAME = N64FlashcartMenu -.DEFAULT_GOAL := $(PROJECT_NAME) +.DEFAULT_GOAL := all SOURCE_DIR = src ASSETS_DIR = assets @@ -9,7 +9,7 @@ OUTPUT_DIR = output include $(N64_INST)/include/n64.mk -N64_CFLAGS += -iquote $(SOURCE_DIR) -I $(SOURCE_DIR)/libs $(FLAGS) +N64_CFLAGS += -iquote $(SOURCE_DIR) -I $(SOURCE_DIR)/libs -flto=auto $(FLAGS) SRCS = \ main.c \ @@ -56,6 +56,7 @@ ASSETS = \ OBJS = $(addprefix $(BUILD_DIR)/, $(addsuffix .o,$(basename $(SRCS) $(ASSETS)))) MINIZ_OBJS = $(filter $(BUILD_DIR)/libs/miniz/%.o,$(OBJS)) SPNG_OBJS = $(filter $(BUILD_DIR)/libs/libspng/%.o,$(OBJS)) +DEPS = $(OBJS:.o=.d) $(MINIZ_OBJS): N64_CFLAGS+=-DMINIZ_NO_TIME -fcompare-debug-second $(SPNG_OBJS): N64_CFLAGS+=-isystem $(SOURCE_DIR)/libs/miniz -DSPNG_USE_MINIZ -fcompare-debug-second @@ -69,34 +70,57 @@ $(BUILD_DIR)/%.o: $(ASSETS_DIR)/%.ttf $(BUILD_DIR)/$(PROJECT_NAME).elf: $(OBJS) +disassembly: $(BUILD_DIR)/$(PROJECT_NAME).elf + @$(N64_OBJDUMP) -S $< > $(BUILD_DIR)/$(PROJECT_NAME).lst +.PHONY: disassembly + $(PROJECT_NAME).z64: N64_ROM_TITLE=$(PROJECT_NAME) -$(PROJECT_NAME): $(PROJECT_NAME).z64 - $(shell mkdir -p $(OUTPUT_DIR)) - $(shell mv $(PROJECT_NAME).z64 $(OUTPUT_DIR)) +$(@info $(shell mkdir -p ./$(OUTPUT_DIR) &> /dev/null)) -sc64_minify: $(PROJECT_NAME) - $(shell python3 ./tools/sc64/minify.py $(BUILD_DIR)/$(PROJECT_NAME).elf $(OUTPUT_DIR)/$(PROJECT_NAME).z64 $(OUTPUT_DIR)/sc64menu.n64) +$(OUTPUT_DIR)/$(PROJECT_NAME).n64: $(PROJECT_NAME).z64 + $(shell mv $< $@) -all: sc64_minify +$(BUILD_DIR)/$(PROJECT_NAME)_stripped.n64: $(OUTPUT_DIR)/$(PROJECT_NAME).n64 + $(shell python3 ./tools/strip_debug_data.py $(BUILD_DIR)/$(PROJECT_NAME).elf $< $@) + @$(N64_CHKSUM) $@ > /dev/null + +64drive: $(OUTPUT_DIR)/$(PROJECT_NAME).n64 + $(shell cp $< $(OUTPUT_DIR)/menu.bin) +.PHONY: 64drive + +ed64: $(BUILD_DIR)/$(PROJECT_NAME)_stripped.n64 + $(shell cp $< $(OUTPUT_DIR)/OS64.v64) +.PHONY: ed64 + +sc64: $(BUILD_DIR)/$(PROJECT_NAME)_stripped.n64 + $(shell cp $< $(OUTPUT_DIR)/sc64menu.n64) +.PHONY: sc64 + +all: $(OUTPUT_DIR)/$(PROJECT_NAME).n64 64drive ed64 sc64 .PHONY: all clean: $(shell rm -rf ./$(BUILD_DIR) ./$(OUTPUT_DIR)) .PHONY: clean -run: $(PROJECT_NAME) +run: $(OUTPUT_DIR)/$(PROJECT_NAME).n64 +ifeq ($(OS),Windows_NT) + ./localdeploy.bat +else ./remotedeploy.sh -# FIXME: improve ability to deploy. -# if devcontainer, use remotedeploy.sh, else -# $(shell sc64deployer --boot direct-rom %~dp0$(OUTPUT_DIR))\$(PROJECT_NAME).z64) +endif .PHONY: run -run-debug: $(PROJECT_NAME) +run-debug: $(OUTPUT_DIR)/$(PROJECT_NAME).n64 +ifeq ($(OS),Windows_NT) + ./localdeploy.bat /d +else ./remotedeploy.sh -d +endif .PHONY: run-debug # test: # TODO: run tests --include $(wildcard $(BUILD_DIR)/*.d) +-include $(DEPS) diff --git a/README.md b/README.md index ecaeccf6..1508fe0f 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ TODO: it does not yet work with `F5`: see https://devblogs.microsoft.com/cppblog WORKAROUND: in the dev container terminal, use make directly, i.e.: `make` The ROM can be found in the `output` directory. -NOTE: a "release" version of the SC64 menu is called `sc64menu.n64` and can be created for when you want to add it directly to the SDCard. This is generated by running `make all` or running `make sc64_minify`. +NOTE: a "release" version of the SC64 menu is called `sc64menu.n64` and can be created for when you want to add it directly to the SDCard. This is generated by running `make all` or running `make sc64`. # Update Libdragon submodule This repo currently uses the `unstable` branch as a submodule at a specific commit. @@ -71,6 +71,7 @@ To update to the latest version, use `git submodule update --remote ` from the t # Generate documentation Run `doxygen` from the dev container terminal. Make sure you fix the warnings before creating a PR! +Generated documentation is located in `output/docs` folder. # OSS licenses used for libraries diff --git a/localdeploy.bat b/localdeploy.bat index 19c6c1fc..f660783e 100644 --- a/localdeploy.bat +++ b/localdeploy.bat @@ -12,7 +12,7 @@ echo: echo: :: Load the ROM echo Loading ROM... -%~dp0tools\sc64\sc64deployer upload %~dp0output\N64FlashcartMenu.z64 +%~dp0tools\sc64\sc64deployer upload %~dp0output\N64FlashcartMenu.n64 echo: echo: @@ -20,6 +20,9 @@ echo: echo !!! Now toggle power to the N64 !!! echo: echo: -::pause -::%~dp0tools\sc64\sc64deployer debug +if not "%1" == "/d" goto :exit + +%~dp0tools\sc64\sc64deployer debug --no-writeback + +:exit diff --git a/remotedeploy.sh b/remotedeploy.sh old mode 100644 new mode 100755 index a233154c..07a8036b --- a/remotedeploy.sh +++ b/remotedeploy.sh @@ -2,7 +2,7 @@ set -e -REMOTE="--remote host.docker.internal:9064" +REMOTE="--remote ${REMOTE:-host.docker.internal:9064}" ## FIXME: this does not work! # Make sure we are connected @@ -17,7 +17,7 @@ echo # Load the ROM echo Loading ROM...: -sc64deployer $REMOTE upload ./output/N64FlashcartMenu.z64 +sc64deployer $REMOTE upload ./output/N64FlashcartMenu.n64 echo echo @@ -27,5 +27,5 @@ echo echo if [ "$1" = "-d" ]; then - sc64deployer $REMOTE debug + sc64deployer $REMOTE debug --no-writeback fi diff --git a/src/flashcart/flashcart.c b/src/flashcart/flashcart.c index 46cb7a4c..3ed28eff 100644 --- a/src/flashcart/flashcart.c +++ b/src/flashcart/flashcart.c @@ -76,7 +76,7 @@ flashcart_error_t flashcart_init (void) { return FLASHCART_ERROR_SD_CARD; } -#ifndef MENU_NO_USB_LOG +#ifndef MENU_RELEASE // NOTE: Some flashcarts doesn't have USB port, can't throw error here debug_init_usblog(); #endif diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 00000000..d2fa938a --- /dev/null +++ b/tools/README.md @@ -0,0 +1,9 @@ +# Source +This Directory. + +# License +See root directory. + +# Description +`strip_debug_data.py` +Removes unnecessary debug data from the end of ROM file to cut on menu load time from SD card. diff --git a/tools/sc64/.gitkeep b/tools/sc64/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tools/sc64/README.md b/tools/sc64/README.md deleted file mode 100644 index d03ac0cd..00000000 --- a/tools/sc64/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Source -This Directory. - -# License -See root directory. - -# Description -`minify.py` -Removes unnecessary null bytes from the end of ROM file to cut on menu load time from SD card. - -# Notes -The deployer `sc64deployer` needs to be downloaded from https://github.com/Polprzewodnikowy/SummerCart64/releases/tag/v2.16.0 and placed in this folder. diff --git a/tools/sc64/minify.py b/tools/sc64/minify.py deleted file mode 100644 index 4bb1bed4..00000000 --- a/tools/sc64/minify.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python3 - -import sys -from subprocess import Popen, PIPE - - -def get_rom_end(elf): - p1 = Popen(f'readelf -s {elf}'.split(), stdout=PIPE) - p2 = Popen('grep -m 1 __rom_end'.split(), stdin=p1.stdout, stdout=PIPE) - stdout, _ = p2.communicate() - rom_end = int(stdout.decode('UTF-8').split()[1], 16) - rom_end &= 0x1FFFFFFF - rom_end -= 0x400 - rom_end += 0x1000 - return rom_end - - -if __name__ == '__main__': - if (len(sys.argv) != 4): - print(f'Usage: python {sys.argv[0]} elf input output') - sys.exit(1) - - elf_file = sys.argv[1] - input_file = sys.argv[2] - output_file = sys.argv[3] - - ALIGN = 512 - - rom_end = get_rom_end(elf_file) - modulo = rom_end % ALIGN - if (modulo > 0): - rom_end += (ALIGN - modulo) - - minified_data = b'' - with open(input_file, 'rb') as f: - minified_data = f.read(rom_end) - - with open(output_file, 'wb') as f: - f.write(minified_data) diff --git a/tools/strip_debug_data.py b/tools/strip_debug_data.py new file mode 100644 index 00000000..f2de0040 --- /dev/null +++ b/tools/strip_debug_data.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 + +import sys +from subprocess import Popen, PIPE + + +def get_symbol_address(elf, symbol): + p1 = Popen(f'readelf -s --wide {elf}'.split(), stdout=PIPE) + p2 = Popen(f'grep -m 1 {symbol}'.split(), stdin=p1.stdout, stdout=PIPE) + stdout, _ = p2.communicate() + + symbol_data = stdout.decode('UTF-8').split() + address = symbol_data[1] + name = symbol_data[7] + + if (symbol != name): + raise Exception(f'Inexact symbol name found [{symbol} != {name}]') + + return int(address, 16) + + +def get_rom_data_end_offset(elf): + ROM_ENTRY_OFFSET = 0x1000 + + libdragon_text_start = get_symbol_address(elf, '__libdragon_text_start') + rom_end = get_symbol_address(elf, '__rom_end') + + return ROM_ENTRY_OFFSET + (rom_end - libdragon_text_start) + + + +if __name__ == '__main__': + if (len(sys.argv) != 4): + print(f'Usage: python {sys.argv[0]} elf input output') + sys.exit(1) + + elf_file = sys.argv[1] + input_file = sys.argv[2] + output_file = sys.argv[3] + + ALIGN = 512 + + try: + length = get_rom_data_end_offset(elf_file) + except Exception as e: + print(e) + sys.exit(2) + + stripped_data = b'' + with open(input_file, 'rb') as f: + stripped_data = f.read(length) + + modulo = (length % ALIGN) + if (modulo > 0): + stripped_data += b'\x00' * (ALIGN - modulo) + + with open(output_file, 'wb') as f: + f.write(stripped_data)