chore: restructured repo to eliminate dist directory

As pointed out in #757, having scripts in `dist` would cause github's code indexing to ignore them.

This also updates packaging tool/scripts, which before wouldn't mark binaries placed inside `uosc.zip` as executable when built on windows. I can't believe there is no CLI tool on windows to do this and I had to write one in go...

.gitignore

@@ -1,3 +1,3 @@
-dist/scripts/uosc/bin
+src/uosc/bin
 release
 *.zip

README.md

@@ -708,6 +708,16 @@ Using `'user'` as `script_id` will overwrite user's `disable_elements` config. E
 
 ## Contributing
 
+### Setup
+
+If you want to test or work on something that involves ziggy (our multitool binary, currently handles searching & downloading subtitles), you first need to build it with:
+
+```
+tools/build ziggy
+```
+
+This requires [`go`](https://go.dev/dl/) to be installed and in path. If you don't want to bother with installing go, and there were no changes to ziggy, you can just use the binaries from [latest release](https://github.com/tomasklaen/uosc/releases/latest/download/uosc.zip). Place folder `scripts/uosc/bin` from `uosc.zip` into `src/uosc/bin`.
+
 ### Localization
 
 If you want to help localizing uosc by either adding a new locale or fixing one that is not up to date, start by running this while in the repository root:
@@ -716,11 +726,11 @@ If you want to help localizing uosc by either adding a new locale or fixing one
 tools/intl languagecode
 ```
 
-`languagecode` can be any existing locale in `dist/scripts/uosc/intl/` directory, or any [IETF language tag](https://en.wikipedia.org/wiki/IETF_language_tag). If it doesn't exist yet, the `intl` tool will create it.
+`languagecode` can be any existing locale in `src/uosc/intl/` directory, or any [IETF language tag](https://en.wikipedia.org/wiki/IETF_language_tag). If it doesn't exist yet, the `intl` tool will create it.
 
 This will parse the codebase for localization strings and use them to either update existing locale by removing unused and setting untranslated strings to `null`, or create a new one with all `null` strings.
 
-You can then navigate to `dist/scripts/uosc/intl/languagecode.json` and start translating.
+You can then navigate to `src/uosc/intl/languagecode.json` and start translating.
 
 ## Why _uosc_?
 

src/tools/lib/zip.go

@@ -0,0 +1,141 @@
+package lib
+
+import (
+	"archive/zip"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+/*
+`files` format:
+
+```
+
+	map[string]string{
+		"/path/on/disk/file1.txt": "file1.txt",
+		"/path/on/disk/file2.txt": "subfolder/file2.txt",
+		"/path/on/disk/file3.txt": "",              // put in root of archive as file3.txt
+		"/path/on/disk/file4.txt": "subfolder/",    // put in subfolder as file4.txt
+		"/path/on/disk/folder":    "Custom Folder", // contents added recursively
+	}
+
+```
+*/
+func ZipFilesWithHeaders(files map[string]string, outputFile string, headerMod HeaderModFn) (ZipStats, error) {
+	path, err := filepath.Abs(outputFile)
+	if err != nil {
+		return ZipStats{}, err
+	}
+	dirname := filepath.Dir(path)
+	err = os.MkdirAll(dirname, os.ModePerm)
+	if err != nil {
+		return ZipStats{}, err
+	}
+
+	f, err := os.Create(path)
+	if err != nil {
+		return ZipStats{}, err
+	}
+	defer f.Close()
+	zw := zip.NewWriter(f)
+	defer zw.Close()
+
+	var filesNum, bytes int64
+
+	addFile := func(srcPath string, nameInArchive string, entry fs.DirEntry) error {
+		src, err := os.Open(srcPath)
+		if err != nil {
+			return err
+		}
+		defer src.Close()
+
+		info, err := entry.Info()
+		if err != nil {
+			return err
+		}
+
+		header, err := zip.FileInfoHeader(info)
+		if err != nil {
+			return err
+		}
+		header.Name = nameInArchive
+		header.Method = zip.Deflate
+		header = headerMod(header)
+		if header.Name == "" {
+			return nil
+		}
+
+		dst, err := zw.CreateHeader(header)
+		if err != nil {
+			return err
+		}
+
+		written, err := io.Copy(dst, src)
+		if err != nil {
+			return err
+		}
+
+		bytes += written
+		filesNum++
+		return nil
+	}
+
+	for src, dst := range files {
+		stat, err := os.Stat(src)
+		if err != nil {
+			return ZipStats{}, err
+		}
+
+		basename := filepath.Base(src)
+		if dst == "" {
+			dst = basename
+		} else if dst[len(dst)-1:] == "/" {
+			dst = dst + basename
+		}
+
+		if !stat.IsDir() {
+			addFile(src, dst, fs.FileInfoToDirEntry(stat))
+			continue
+		}
+
+		err = filepath.WalkDir(src, func(path string, entry fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			if entry.IsDir() {
+				return nil
+			}
+
+			relativePath, err := filepath.Rel(src, path)
+			if err != nil {
+				return err
+			}
+
+			err = addFile(path, dst+"/"+filepath.ToSlash(relativePath), entry)
+			if err != nil {
+				return err
+			}
+
+			return nil
+		})
+
+		if err != nil {
+			return ZipStats{}, err
+		}
+	}
+
+	return ZipStats{
+		FilesNum: filesNum,
+		Bytes:    bytes,
+	}, nil
+}
+
+// If `HeaderModFn` function sets `header.Name` to empty string, file will be skipped.
+type HeaderModFn func(header *zip.FileHeader) *zip.FileHeader
+
+type ZipStats struct {
+	FilesNum int64
+	Bytes    int64
+}

src/tools/tools.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"uosc/bins/src/tools/tools"
+)
+
+func main() {
+	command := "help"
+
+	if len(os.Args) > 1 {
+		command = os.Args[1]
+	}
+
+	switch command {
+	case "intl":
+		tools.Intl(os.Args[2:])
+
+	case "package":
+		tools.Packager(os.Args[2:])
+
+	// Help
+	default:
+		fmt.Printf(`uosc tools.
+
+Usage:
+
+  tools <command> [args]
+
+Available <command>s:
+
+  intl - localization helper
+  package - package uosc release files
+
+Run 'tools <command> -h/--help' for help on how to use each tool.
+`)
+	}
+}

src/tools/tools/base.go

@@ -0,0 +1,12 @@
+package tools
+
+func check(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func must[T any](t T, err error) T {
+	check(err)
+	return t
+}

src/tools/tools/intl.go

@@ -1,4 +1,4 @@
-package main
+package tools
 
 import (
 	"bufio"
@@ -15,10 +15,10 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 )
 
-func main() {
+func Intl(args []string) {
 	cwd, err := os.Getwd()
 	check(err)
-	uoscRootRelative := "dist/scripts/uosc"
+	uoscRootRelative := "src/uosc"
 	intlRootRelative := uoscRootRelative + "/intl"
 	uoscRoot := filepath.Join(cwd, uoscRootRelative)
 
@@ -29,7 +29,7 @@ func main() {
 	}
 
 	// Help
-	if len(os.Args) <= 1 || len(os.Args) > 1 && sets.New("--help", "-h").Has(os.Args[1]) {
+	if len(args) < 1 || len(args) > 0 && sets.New("--help", "-h").Has(args[0]) {
 		fmt.Printf(`Updates or creates a localization files by parsing the codebase for localization strings, and (re)constructing the locale files with them.
 Strings no longer in use are removed. Strings not yet translated are set to "null".
 
@@ -58,15 +58,14 @@ Examples:
 	}
 
 	var locales []string
-	if os.Args[1] == "all" {
+	if args[0] == "all" {
 		intlRoot := filepath.Join(cwd, intlRootRelative)
 		locales = must(listFilenamesOfType(intlRoot, ".json"))
 	} else {
-		locales = strings.Split(os.Args[1], ",")
+		locales = strings.Split(args[0], ",")
 	}
 
 	holePunchLocales(locales, uoscRoot)
-
 }
 
 func holePunchLocales(locales []string, rootPath string) {
@@ -237,17 +236,6 @@ func holePunchLocales(locales []string, rootPath string) {
 	}
 }
 
-func check(err error) {
-	if err != nil {
-		panic(err)
-	}
-}
-
-func must[T any](t T, err error) T {
-	check(err)
-	return t
-}
-
 func listFilenamesOfType(directoryPath string, extension string) ([]string, error) {
 	files := []string{}
 	extension = strings.ToLower(extension)

src/tools/tools/packager.go

@@ -0,0 +1,65 @@
+package tools
+
+import (
+	"archive/zip"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"uosc/bins/src/tools/lib"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+func Packager(args []string) {
+	// Display help.
+	if len(args) > 0 && sets.New("--help", "-h").Has(args[0]) {
+		fmt.Printf(`Packages uosc release files into 'release/' directory, while ensuring binaries inside the zip file are marked as executable even when packaged on windows (otherwise this could've just be a simple .ps1/.sh file).`)
+		os.Exit(0)
+	}
+
+	cwd := must(os.Getwd())
+	releaseRoot := filepath.Join(cwd, "release")
+	releaseArchiveSrcDstMap := map[string]string{
+		filepath.Join(cwd, "src/fonts"): "",
+		filepath.Join(cwd, "src/uosc"):  "scripts/",
+	}
+	releaseConfigPath := filepath.Join(releaseRoot, "uosc.conf")
+	releaseArchivePath := filepath.Join(releaseRoot, "uosc.zip")
+	sourceConfigPath := filepath.Join(cwd, "src/uosc.conf")
+
+	// Naive check binaries are built.
+	bins := must(os.ReadDir(filepath.Join(cwd, "src/uosc/bin")))
+	if len(bins) == 0 {
+		check(errors.New("binaries are not built ('src/uosc/bin' is empty)"))
+	}
+
+	// Cleanup old release.
+	check(os.RemoveAll(releaseRoot))
+
+	// Package new release
+	var modHeaders lib.HeaderModFn = func(header *zip.FileHeader) *zip.FileHeader {
+		// Mark binaries as executable.
+		if strings.HasPrefix(header.Name, "scripts/uosc/bin/") {
+			header.SetMode(0755)
+		}
+		return header
+	}
+	stats := must(lib.ZipFilesWithHeaders(releaseArchiveSrcDstMap, releaseArchivePath, modHeaders))
+
+	// Copy config to release folder for convenience.
+	configFileSrc := must(os.Open(sourceConfigPath))
+	configFileDst := must(os.Create(releaseConfigPath))
+	confSize := must(io.Copy(configFileDst, configFileSrc))
+
+	fmt.Printf(
+		"Packaging into: %s\n- uosc.zip:  %.2f MB, %d files\n- uosc.conf: %.1f KB",
+		filepath.ToSlash(must(filepath.Rel(cwd, releaseRoot)))+"/",
+		float64(stats.Bytes)/1024/1024,
+		stats.FilesNum,
+		float64(confSize)/1024,
+	)
+}

tools/build

@@ -2,7 +2,7 @@
 # Script to build one of uosc binaries.
 # Requirements: go, upx (if compressing)
 # Usage: tools/build <name> [-c]
-# <name> can be: intl, ziggy
+# <name> can be: tools, ziggy
 # -c enables binary compression with upx (only needed for builds being released)
 
 abort() {
@@ -14,28 +14,28 @@ if [ ! -d "$PWD/src" ]; then
 	abort "'src' directory not found. Make sure this script is run in uosc's repository root as current working directory."
 fi
 
-if [ "$1" = "intl" ]; then
+if [ "$1" = "tools" ]; then
 	export GOARCH="amd64"
-	src="./src/intl/intl.go"
+	src="./src/tools/tools.go"
 	out_dir="./tools"
 
 	echo "Building for Windows..."
 	export GOOS="windows"
-	go build -ldflags "-s -w" -o "$out_dir/intl.exe" $src
+	go build -ldflags "-s -w" -o "$out_dir/tools.exe" $src
 
 	echo "Building for Linux..."
 	export GOOS="linux"
-	go build -ldflags "-s -w" -o "$out_dir/intl-linux" $src
+	go build -ldflags "-s -w" -o "$out_dir/tools-linux" $src
 
 	echo "Building for MacOS..."
 	export GOOS="darwin"
-	go build -ldflags "-s -w" -o "$out_dir/intl-darwin" $src
+	go build -ldflags "-s -w" -o "$out_dir/tools-darwin" $src
 
 	if [ "$2" = "-c" ]; then
 		echo "Compressing binaries..."
-		upx --brute "$out_dir/intl.exe"
-		upx --brute "$out_dir/intl-linux"
-		upx --brute "$out_dir/intl-darwin"
+		upx --brute "$out_dir/tools.exe"
+		upx --brute "$out_dir/tools-linux"
+		upx --brute "$out_dir/tools-darwin"
 	fi
 
 	unset GOARCH
@@ -76,6 +76,6 @@ else
 	echo "Tool to build one of uosc binaries. Requires go to be installed and in path."
 	echo "Requirements: go, upx (if compressing)"
 	echo "Usage: tools/build <name> [-c]"
-	echo "<name> can be: intl, ziggy"
+	echo "<name> can be: tools, ziggy"
 	echo "-c enables binary compression (requires upx)"
 fi

tools/build.ps1

@@ -14,28 +14,28 @@ if (!(Test-Path -Path "$PWD/src" -PathType Container)) {
 	Abort("'src' directory not found. Make sure this script is run in uosc's repository root as current working directory.")
 }
 
-if ($args[0] -eq "intl") {
+if ($args[0] -eq "tools") {
 	$env:GOARCH = "amd64"
-	$Src = "./src/intl/intl.go"
+	$Src = "./src/tools/tools.go"
 	$OutDir = "./tools"
 
 	Write-Output "Building for Windows..."
 	$env:GOOS = "windows"
-	go build -ldflags "-s -w" -o "$OutDir/intl.exe" $Src
+	go build -ldflags "-s -w" -o "$OutDir/tools.exe" $Src
 
 	Write-Output "Building for Linux..."
 	$env:GOOS = "linux"
-	go build -ldflags "-s -w" -o "$OutDir/intl-linux" $Src
+	go build -ldflags "-s -w" -o "$OutDir/tools-linux" $Src
 
 	Write-Output "Building for MacOS..."
 	$env:GOOS = "darwin"
-	go build -ldflags "-s -w" -o "$OutDir/intl-darwin" $Src
+	go build -ldflags "-s -w" -o "$OutDir/tools-darwin" $Src
 
 	if ($args[1] -eq "-c") {
 		Write-Output "Compressing binaries..."
-		upx --brute "$OutDir/intl.exe"
-		upx --brute "$OutDir/intl-linux"
-		upx --brute "$OutDir/intl-darwin"
+		upx --brute "$OutDir/tools.exe"
+		upx --brute "$OutDir/tools-linux"
+		upx --brute "$OutDir/tools-darwin"
 	}
 
 	Remove-Item Env:\GOOS
@@ -44,7 +44,7 @@ if ($args[0] -eq "intl") {
 elseif ($args[0] -eq "ziggy") {
 	$env:GOARCH = "amd64"
 	$Src = "./src/ziggy/ziggy.go"
-	$OutDir = "./dist/scripts/uosc/bin"
+	$OutDir = "./src/uosc/bin"
 
 	if (!(Test-Path $OutDir)) {
 		New-Item -ItemType Directory -Force -Path $OutDir > $null

tools/intl

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 if [ "$(uname)" == "Darwin" ]; then
-	"$SCRIPT_DIR/intl-darwin" $*
+	"$SCRIPT_DIR/tools-darwin" intl $*
 else
-	"$SCRIPT_DIR/intl-linux" $*
+	"$SCRIPT_DIR/tools-linux" intl $*
 fi

tools/intl.ps1

@@ -0,0 +1 @@
+& "$PSScriptRoot/tools.exe" intl $args

tools/package

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+if [ "$(uname)" == "Darwin" ]; then
+	"$SCRIPT_DIR/tools-darwin" package $*
+else
+	"$SCRIPT_DIR/tools-linux" package $*
+fi

tools/package.ps1

@@ -1,43 +1 @@
-# Package uosc release.
-
-Function Abort($Message) {
-	Write-Output "Error: $Message"
-	Write-Output "Aborting!"
-	Exit 1
-}
-
-Function DeleteIfExists($Path) {
-	if (Test-Path $Path) {
-		Remove-Item -LiteralPath $Path -Force -Recurse > $null
-	}
-}
-
-if (!(Test-Path -Path "$PWD/src" -PathType Container)) {
-	Abort("'src' directory not found. Make sure this script is run in uosc's repository root as current working directory.")
-}
-
-if (!(Test-Path -Path "$PWD/dist/scripts/uosc/bin/ziggy-linux" -PathType Leaf)) {
-	Abort("'dist/scripts/uosc/bin' binaries are not build.")
-}
-
-$ReleaseDir = "release"
-
-# Clear old
-DeleteIfExists($ReleaseDir)
-if (!(Test-Path $ReleaseDir)) {
-	try {
-		New-Item -ItemType Directory -Force -Path $ReleaseDir > $null
-	}
-	catch {
-		Abort("Couldn't create release directory.")
-	}
-}
-
-# Package new
-$compress = @{
-	LiteralPath = "dist/fonts", "dist/scripts"
-	CompressionLevel = "Optimal"
-	DestinationPath = "$ReleaseDir/uosc.zip"
-}
-Compress-Archive @compress
-Copy-Item "dist/script-opts/uosc.conf" -Destination $ReleaseDir
+& "$PSScriptRoot/tools.exe" package $args