mirror of
https://github.com/Belphemur/CBZOptimizer.git
synced 2026-07-22 11:25:40 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e87e73eb28 | ||
|
|
d7782fdb25 | ||
|
|
1d981be21b | ||
|
|
0b3c3fa90c | ||
|
|
52c93acd35 | ||
|
|
ab1e2ff05c | ||
|
|
eca9671881 | ||
|
|
40922078d5 | ||
|
|
efb4dce1b7 | ||
|
|
30d0d632bf | ||
|
|
1e692ed19b | ||
|
|
69c1a37577 | ||
|
|
e1e8ea92c1 | ||
|
|
c2f809ed2c | ||
|
|
f2a65425cf | ||
|
|
20de3ed0ae | ||
|
|
b9170b14a1 | ||
|
|
18ff04c81f | ||
|
|
2663f17be3 | ||
|
|
dc9d33bbca | ||
|
|
cc7da1ef3b | ||
|
|
8c538ce95b | ||
|
|
c8524e9b5f | ||
|
|
cbad6e99ac | ||
|
|
65b9961856 | ||
|
|
d044573019 | ||
|
|
964a6ea295 | ||
|
|
02b15c8124 | ||
|
|
deeb30aa24 | ||
|
|
ad24ab68ac | ||
|
|
d78b523ae7 | ||
|
|
7b1fb13a21 | ||
|
|
414a43476b | ||
|
|
dbac6c20ff | ||
|
|
ef1a658028 | ||
|
|
76aea38ff2 | ||
|
|
a69eb9d8a5 | ||
|
|
a6875c3a12 | ||
|
|
b3686e17d8 | ||
|
|
6ea989eedd | ||
|
|
f104add37a | ||
|
|
39632906da | ||
|
|
1be0978e89 | ||
|
|
71bdf7d111 | ||
|
|
3123fa27f9 | ||
|
|
7cd711ab5a | ||
|
|
df3206b80c | ||
|
|
878da22fdb | ||
|
|
fcf6bf514b | ||
|
|
70a3306ca9 | ||
|
|
0ff3eab0b9 | ||
|
|
45851713c1 | ||
|
|
cca06642f8 | ||
|
|
a0a79c6439 | ||
|
|
6c45a8507f | ||
|
|
d66d445bbb | ||
|
|
aabb0e2a02 | ||
|
|
8349569e33 | ||
|
|
b434a18802 | ||
|
|
4057cfb6c9 | ||
|
|
617ee8a0c3 | ||
|
|
f37ca7e4e7 | ||
|
|
38c8f2a9cb | ||
|
|
03479c8772 | ||
|
|
ee47432721 | ||
|
|
5a0fe68e68 | ||
|
|
e535809e74 | ||
|
|
af5bfe8000 | ||
|
|
9ac9901990 | ||
|
|
8c52010dfe | ||
|
|
aefadafc7d | ||
|
|
ba1ab20697 | ||
|
|
43593c37fc | ||
|
|
44a4726258 | ||
|
|
e71a3d7693 | ||
|
|
992e37f9af | ||
|
|
a2f6805d47 | ||
|
|
552364f69c | ||
|
|
da65eeecba | ||
|
|
5d35a2e3fa | ||
|
|
1568334c03 | ||
|
|
31ef12bb17 | ||
|
|
9529004554 | ||
|
|
6a2efc42ac | ||
|
|
44e2469e34 | ||
|
|
9b6a733012 | ||
|
|
b80535d211 | ||
|
|
3a2fb2a97e | ||
|
|
c5de49a310 | ||
|
|
cd0f056648 | ||
|
|
a2feca6cca | ||
|
|
1fa54e1936 | ||
|
|
ce8aaba165 | ||
|
|
647b139ea0 | ||
|
|
16b3ce3c9b | ||
|
|
8d359aa575 | ||
|
|
97f89a51c6 | ||
|
|
6840de3a89 | ||
|
|
117b55eeaf | ||
|
|
287ae8df8b | ||
|
|
481da7c769 | ||
|
|
e269537049 | ||
|
|
cc4829cb39 | ||
|
|
65747d35c0 | ||
|
|
eb8803302c | ||
|
|
e60e30f5a0 | ||
|
|
7f5f690e66 | ||
|
|
f752586432 | ||
|
|
9a72d64a38 | ||
|
|
09655e225c | ||
|
|
90d75361a7 | ||
|
|
503fad46a6 | ||
|
|
e842b49535 | ||
|
|
86d20e14b1 | ||
|
|
7081f4aa1c | ||
|
|
6d8e1e2f5e | ||
|
|
77279cb0c5 | ||
|
|
82ab972c2e | ||
|
|
ae754ae5d8 | ||
|
|
507d8df103 | ||
|
|
545382c887 | ||
|
|
255b158778 | ||
|
|
4f9dacdaf6 | ||
|
|
3e62ab40e3 | ||
|
|
51af843432 | ||
|
|
6b92336ba1 | ||
|
|
a6ad1dada3 | ||
|
|
17fe01f27c | ||
|
|
4fa3014d80 | ||
|
|
a47af5a7a8 | ||
|
|
d7f13132f4 | ||
|
|
a8587f3f1f |
@@ -0,0 +1 @@
|
|||||||
|
testdata/large/*.cbz filter=lfs diff=lfs merge=lfs -text
|
||||||
@@ -0,0 +1,328 @@
|
|||||||
|
# CBZOptimizer - GitHub Copilot Instructions
|
||||||
|
|
||||||
|
## Project Overview
|
||||||
|
|
||||||
|
CBZOptimizer is a Go-based command-line tool designed to optimize CBZ (Comic Book Zip) and CBR (Comic Book RAR) files by converting images to modern formats (primarily WebP) with configurable quality settings. The tool reduces the size of comic book archives while maintaining acceptable image quality.
|
||||||
|
|
||||||
|
**Key Features:**
|
||||||
|
- Convert CBZ/CBR files to optimized CBZ format
|
||||||
|
- WebP image encoding with quality control
|
||||||
|
- Parallel chapter processing
|
||||||
|
- File watching for automatic optimization
|
||||||
|
- Optional page splitting for large images
|
||||||
|
- Timeout handling for problematic files
|
||||||
|
|
||||||
|
## Technology Stack
|
||||||
|
|
||||||
|
- **Language:** Go 1.25+
|
||||||
|
- **CLI Framework:** Cobra + Viper
|
||||||
|
- **Logging:** zerolog (structured logging)
|
||||||
|
- **Image Processing:** go-webpbin/v2 for WebP encoding
|
||||||
|
- **Archive Handling:** mholt/archives for CBZ/CBR processing
|
||||||
|
- **Testing:** testify + gotestsum
|
||||||
|
|
||||||
|
## Project Structure
|
||||||
|
|
||||||
|
```
|
||||||
|
.
|
||||||
|
├── cmd/
|
||||||
|
│ ├── cbzoptimizer/ # Main CLI application
|
||||||
|
│ │ ├── commands/ # Cobra commands (optimize, watch)
|
||||||
|
│ │ └── main.go # Entry point
|
||||||
|
│ └── encoder-setup/ # WebP encoder setup utility
|
||||||
|
│ └── main.go # Encoder initialization (build tag: encoder_setup)
|
||||||
|
├── internal/
|
||||||
|
│ ├── cbz/ # CBZ/CBR file operations
|
||||||
|
│ │ ├── cbz_loader.go # Load and parse comic archives
|
||||||
|
│ │ └── cbz_creator.go # Create optimized archives
|
||||||
|
│ ├── manga/ # Domain models
|
||||||
|
│ │ ├── chapter.go # Chapter representation
|
||||||
|
│ │ ├── page.go # Page image handling
|
||||||
|
│ │ └── page_container.go # Page collection management
|
||||||
|
│ └── utils/ # Utility functions
|
||||||
|
│ ├── optimize.go # Core optimization logic
|
||||||
|
│ └── errs/ # Error handling utilities
|
||||||
|
└── pkg/
|
||||||
|
└── converter/ # Image conversion abstractions
|
||||||
|
├── converter.go # Converter interface
|
||||||
|
├── webp/ # WebP implementation
|
||||||
|
│ ├── webp_converter.go # WebP conversion logic
|
||||||
|
│ └── webp_provider.go # WebP encoder provider
|
||||||
|
├── errors/ # Conversion error types
|
||||||
|
└── constant/ # Shared constants
|
||||||
|
```
|
||||||
|
|
||||||
|
## Building and Testing
|
||||||
|
|
||||||
|
### Prerequisites
|
||||||
|
|
||||||
|
Before building or testing, the WebP encoder must be set up:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Build the encoder-setup utility
|
||||||
|
go build -tags encoder_setup -o encoder-setup ./cmd/encoder-setup
|
||||||
|
|
||||||
|
# Run encoder setup (downloads and configures libwebp 1.6.0)
|
||||||
|
./encoder-setup
|
||||||
|
```
|
||||||
|
|
||||||
|
This step is **required** before running tests or building the main application.
|
||||||
|
|
||||||
|
### Build Commands
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Build the main application
|
||||||
|
go build -o cbzconverter ./cmd/cbzoptimizer
|
||||||
|
|
||||||
|
# Build with version information
|
||||||
|
go build -ldflags "-s -w -X main.version=1.0.0 -X main.commit=abc123 -X main.date=2024-01-01" -o cbzconverter ./cmd/cbzoptimizer
|
||||||
|
```
|
||||||
|
|
||||||
|
### Testing
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Install test runner
|
||||||
|
go install gotest.tools/gotestsum@latest
|
||||||
|
|
||||||
|
# Run all tests with coverage
|
||||||
|
gotestsum --format testname -- -race -coverprofile=coverage.txt -covermode=atomic ./...
|
||||||
|
|
||||||
|
# Run specific package tests
|
||||||
|
go test -v ./internal/cbz/...
|
||||||
|
go test -v ./pkg/converter/...
|
||||||
|
|
||||||
|
# Run integration tests
|
||||||
|
go test -v ./internal/utils/...
|
||||||
|
```
|
||||||
|
|
||||||
|
A large-file integration test (`TestOptimizeIntegration_LargeFile`) exercises the
|
||||||
|
optimize pipeline against a ~1GB CBZ fixture stored via Git LFS
|
||||||
|
(`testdata/large/large_chapter.cbz`, tracked in `.gitattributes`). It is skipped
|
||||||
|
by default; fetch the fixture with `git lfs pull` and opt in with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
CBZ_RUN_LARGE_FILE_TEST=1 go test -v ./internal/utils/... -run TestOptimizeIntegration_LargeFile
|
||||||
|
```
|
||||||
|
|
||||||
|
### Linting
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Install golangci-lint if not available
|
||||||
|
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
|
||||||
|
|
||||||
|
# Run linter
|
||||||
|
golangci-lint run
|
||||||
|
```
|
||||||
|
|
||||||
|
## Code Conventions
|
||||||
|
|
||||||
|
### Go Style
|
||||||
|
|
||||||
|
- **Follow standard Go conventions:** Use `gofmt` and `goimports`
|
||||||
|
- **Package naming:** Short, lowercase, single-word names
|
||||||
|
- **Error handling:** Always check errors explicitly; use structured error wrapping with `fmt.Errorf("context: %w", err)`
|
||||||
|
- **Context usage:** Pass `context.Context` as first parameter for operations that may be cancelled
|
||||||
|
|
||||||
|
### Commit Messages
|
||||||
|
|
||||||
|
This project follows [Conventional Commits](https://www.conventionalcommits.org/). Prefix commit messages with a type (and optional scope), for example:
|
||||||
|
|
||||||
|
- `fix(watch): debounce fsnotify events before optimizing`
|
||||||
|
- `feat(converter): add avif support`
|
||||||
|
- `docs: clarify commit convention`
|
||||||
|
- `chore: update dependency`
|
||||||
|
|
||||||
|
### Logging
|
||||||
|
|
||||||
|
Use **zerolog** for all logging:
|
||||||
|
|
||||||
|
```go
|
||||||
|
import "github.com/rs/zerolog/log"
|
||||||
|
|
||||||
|
// Info level with structured fields
|
||||||
|
log.Info().Str("file", path).Int("pages", count).Msg("Processing file")
|
||||||
|
|
||||||
|
// Debug level for detailed diagnostics
|
||||||
|
log.Debug().Str("file", path).Uint8("quality", quality).Msg("Optimization parameters")
|
||||||
|
|
||||||
|
// Error level with error wrapping
|
||||||
|
log.Error().Str("file", path).Err(err).Msg("Failed to load chapter")
|
||||||
|
```
|
||||||
|
|
||||||
|
**Log Levels (in order of verbosity):**
|
||||||
|
- `panic` - System panic conditions
|
||||||
|
- `fatal` - Fatal errors requiring exit
|
||||||
|
- `error` - Error conditions
|
||||||
|
- `warn` - Warning conditions
|
||||||
|
- `info` - General information (default)
|
||||||
|
- `debug` - Debug-level messages
|
||||||
|
- `trace` - Trace-level messages
|
||||||
|
|
||||||
|
### Error Handling
|
||||||
|
|
||||||
|
- Use the custom `errs` package for deferred error handling:
|
||||||
|
```go
|
||||||
|
import "github.com/belphemur/CBZOptimizer/v2/internal/utils/errs"
|
||||||
|
|
||||||
|
func processFile() (err error) {
|
||||||
|
defer errs.Wrap(&err, "failed to process file")
|
||||||
|
// ... implementation
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- Define custom error types in `pkg/converter/errors/` for specific error conditions
|
||||||
|
- Always provide context when wrapping errors
|
||||||
|
|
||||||
|
### Testing
|
||||||
|
|
||||||
|
- Use **testify** for assertions:
|
||||||
|
```go
|
||||||
|
import "github.com/stretchr/testify/assert"
|
||||||
|
|
||||||
|
func TestSomething(t *testing.T) {
|
||||||
|
result, err := DoSomething()
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Equal(t, expected, result)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- Use table-driven tests for multiple scenarios:
|
||||||
|
```go
|
||||||
|
testCases := []struct {
|
||||||
|
name string
|
||||||
|
input string
|
||||||
|
expected string
|
||||||
|
expectError bool
|
||||||
|
}{
|
||||||
|
{"case1", "input1", "output1", false},
|
||||||
|
{"case2", "input2", "output2", true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range testCases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
// test implementation
|
||||||
|
})
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- Integration tests should be in `*_integration_test.go` files
|
||||||
|
- Use temporary directories for file operations in tests
|
||||||
|
|
||||||
|
### Command Structure (Cobra)
|
||||||
|
|
||||||
|
- Commands are in `cmd/cbzoptimizer/commands/`
|
||||||
|
- Each command is in its own file (e.g., `optimize_command.go`, `watch_command.go`)
|
||||||
|
- Use Cobra's persistent flags for global options
|
||||||
|
- Use Viper for configuration management
|
||||||
|
|
||||||
|
### Dependencies
|
||||||
|
|
||||||
|
**Key external packages:**
|
||||||
|
- `github.com/belphemur/go-webpbin/v2` - WebP encoding (libwebp wrapper)
|
||||||
|
- `github.com/mholt/archives` - Archive format handling
|
||||||
|
- `github.com/spf13/cobra` - CLI framework
|
||||||
|
- `github.com/spf13/viper` - Configuration management
|
||||||
|
- `github.com/rs/zerolog` - Structured logging
|
||||||
|
- `github.com/oliamb/cutter` - Image cropping for page splitting
|
||||||
|
- `golang.org/x/image` - Extended image format support
|
||||||
|
|
||||||
|
## Docker Considerations
|
||||||
|
|
||||||
|
The Dockerfile uses a multi-stage build and requires:
|
||||||
|
1. The compiled `CBZOptimizer` binary (from goreleaser)
|
||||||
|
2. The `encoder-setup` binary (built with `-tags encoder_setup`)
|
||||||
|
3. The encoder-setup is run during image build to configure WebP encoder
|
||||||
|
|
||||||
|
The encoder must be set up in the container before the application runs.
|
||||||
|
|
||||||
|
## Common Tasks
|
||||||
|
|
||||||
|
### Adding a New Command
|
||||||
|
|
||||||
|
1. Create `cmd/cbzoptimizer/commands/newcommand_command.go`
|
||||||
|
2. Define the command using Cobra:
|
||||||
|
```go
|
||||||
|
var newCmd = &cobra.Command{
|
||||||
|
Use: "new",
|
||||||
|
Short: "Description",
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
// implementation
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
rootCmd.AddCommand(newCmd)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
3. Add tests in `newcommand_command_test.go`
|
||||||
|
|
||||||
|
### Adding a New Image Format Converter
|
||||||
|
|
||||||
|
1. Create a new package under `pkg/converter/` (e.g., `avif/`)
|
||||||
|
2. Implement the `Converter` interface from `pkg/converter/converter.go`
|
||||||
|
3. Add tests following existing patterns in `pkg/converter/webp/`
|
||||||
|
4. Update command flags to support the new format
|
||||||
|
|
||||||
|
### Modifying Optimization Logic
|
||||||
|
|
||||||
|
The core optimization logic is in `internal/utils/optimize.go`:
|
||||||
|
- Uses the `OptimizeOptions` struct for parameters
|
||||||
|
- Handles chapter loading, conversion, and saving
|
||||||
|
- Implements timeout handling with context
|
||||||
|
- Provides structured logging at each step
|
||||||
|
|
||||||
|
## CI/CD
|
||||||
|
|
||||||
|
### GitHub Actions Workflows
|
||||||
|
|
||||||
|
1. **test.yml** - Runs on every push/PR
|
||||||
|
- Sets up Go environment
|
||||||
|
- Runs encoder-setup
|
||||||
|
- Executes tests with coverage
|
||||||
|
- Uploads results to Codecov
|
||||||
|
|
||||||
|
2. **release.yml** - Runs on version tags
|
||||||
|
- Uses goreleaser for multi-platform builds
|
||||||
|
- Builds Docker images for linux/amd64 and linux/arm64
|
||||||
|
- Signs releases with cosign
|
||||||
|
- Generates SBOMs with syft
|
||||||
|
|
||||||
|
3. **qodana.yml** - Code quality analysis
|
||||||
|
|
||||||
|
### Release Process
|
||||||
|
|
||||||
|
Releases are automated via goreleaser:
|
||||||
|
- Tag format: `v*` (e.g., `v2.1.0`)
|
||||||
|
- Builds for: linux, darwin, windows (amd64, arm64)
|
||||||
|
- Creates Docker images and pushes to ghcr.io
|
||||||
|
- Generates checksums and SBOMs
|
||||||
|
|
||||||
|
## Performance Considerations
|
||||||
|
|
||||||
|
- **Parallelism:** Use `--parallelism` flag to control concurrent chapter processing. Regardless of this value, the total number of pages converted at the same time (i.e. concurrent `cwebp` processes) is capped process-wide to the number of CPU cores, so raising parallelism spreads that budget across more chapters instead of multiplying resource usage.
|
||||||
|
- **Memory:** Original page images are loaded in memory, but converted pages are staged to a temporary folder on disk as soon as they're produced (see `manga.Page.TempFilePath` / `manga.Chapter.TempDir`) instead of being held in memory until the whole chapter is written out. The staging folder is cleaned up once the chapter has been written to its final CBZ file.
|
||||||
|
- **Disk:** Since converted pages are staged on disk, ensure enough free space (roughly the size of the converted chapter) is available in the OS temp directory.
|
||||||
|
- **Timeouts:** Use `--timeout` flag to prevent hanging on problematic files
|
||||||
|
- **WebP Quality:** Balance quality (0-100) vs file size; default is 85
|
||||||
|
|
||||||
|
## Security
|
||||||
|
|
||||||
|
- No credentials or secrets should be committed
|
||||||
|
- Archive extraction includes path traversal protection
|
||||||
|
- File permissions are preserved during operations
|
||||||
|
- Docker images run as non-root user (`abc`, UID 99)
|
||||||
|
|
||||||
|
## Additional Notes
|
||||||
|
|
||||||
|
- CBR files are always converted to CBZ format (RAR is read-only)
|
||||||
|
- The `--override` flag deletes the original file after successful conversion
|
||||||
|
- Page splitting is useful for double-page spreads or very tall images
|
||||||
|
- Watch mode uses fsnotify-backed recursive directory monitoring on Linux
|
||||||
|
- Bash completion is available via `cbzconverter completion bash`
|
||||||
|
|
||||||
|
## Getting Help
|
||||||
|
|
||||||
|
- Use `--help` flag for command documentation
|
||||||
|
- Use `--log debug` for detailed diagnostic output
|
||||||
|
- Check GitHub Issues for known problems
|
||||||
|
- Review test files for usage examples
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
name: Copilot Setup Steps
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
copilot-setup-steps:
|
||||||
|
name: Setup Go and gopls
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
|
- name: Set up Go
|
||||||
|
uses: actions/setup-go@v6
|
||||||
|
with:
|
||||||
|
go-version-file: go.mod
|
||||||
|
cache: true
|
||||||
|
|
||||||
|
- name: Verify Go installation
|
||||||
|
run: |
|
||||||
|
go version
|
||||||
|
go env
|
||||||
|
|
||||||
|
- name: Install gopls
|
||||||
|
run: |
|
||||||
|
go install golang.org/x/tools/gopls@latest
|
||||||
|
|
||||||
|
- name: Verify gopls installation
|
||||||
|
run: |
|
||||||
|
gopls version
|
||||||
|
|
||||||
|
- name: Install golangci-lint
|
||||||
|
uses: golangci/golangci-lint-action@v9
|
||||||
|
with:
|
||||||
|
version: latest
|
||||||
|
|
||||||
|
- name: Download Go dependencies
|
||||||
|
run: |
|
||||||
|
go mod download
|
||||||
|
go mod verify
|
||||||
|
|
||||||
|
- name: Build encoder-setup utility
|
||||||
|
run: |
|
||||||
|
go build -tags encoder_setup -o encoder-setup ./cmd/encoder-setup
|
||||||
|
ls -lh encoder-setup
|
||||||
|
|
||||||
|
- name: Run encoder-setup
|
||||||
|
run: |
|
||||||
|
./encoder-setup
|
||||||
|
|
||||||
|
- name: Install gotestsum
|
||||||
|
run: |
|
||||||
|
go install gotest.tools/gotestsum@latest
|
||||||
|
|
||||||
|
- name: Verify gotestsum installation
|
||||||
|
run: |
|
||||||
|
gotestsum --version
|
||||||
|
|
||||||
|
- name: Setup complete
|
||||||
|
run: |
|
||||||
|
echo "✅ Go environment setup complete"
|
||||||
|
echo "✅ gopls (Go language server) installed"
|
||||||
|
echo "✅ golangci-lint installed"
|
||||||
|
echo "✅ Dependencies downloaded and verified"
|
||||||
|
echo "✅ WebP encoder configured (libwebp 1.6.0)"
|
||||||
|
echo "✅ gotestsum (test runner) installed"
|
||||||
@@ -1,23 +0,0 @@
|
|||||||
name: Qodana
|
|
||||||
on:
|
|
||||||
workflow_dispatch:
|
|
||||||
pull_request:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
qodana:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
permissions:
|
|
||||||
contents: write
|
|
||||||
pull-requests: write
|
|
||||||
checks: write
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v5
|
|
||||||
with:
|
|
||||||
ref: ${{ github.event.pull_request.head.sha }} # to check out the actual pull request commit, not the merge commit
|
|
||||||
fetch-depth: 0 # a full history is required for pull request analysis
|
|
||||||
- name: 'Qodana Scan'
|
|
||||||
uses: JetBrains/qodana-action@v2024.1
|
|
||||||
env:
|
|
||||||
QODANA_TOKEN: ${{ secrets.QODANA_TOKEN }}
|
|
||||||
@@ -20,7 +20,7 @@ jobs:
|
|||||||
release:
|
release:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@v7
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0 # this is important, otherwise it won't checkout the full tree (i.e. no previous tags)
|
fetch-depth: 0 # this is important, otherwise it won't checkout the full tree (i.e. no previous tags)
|
||||||
- name: Set up Go
|
- name: Set up Go
|
||||||
@@ -28,28 +28,32 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
go-version-file: go.mod
|
go-version-file: go.mod
|
||||||
cache: true
|
cache: true
|
||||||
- uses: sigstore/cosign-installer@v3.9.2 # installs cosign
|
- name: Install Syft
|
||||||
- uses: anchore/sbom-action/download-syft@v0.20.5 # installs syft
|
uses: anchore/sbom-action/download-syft@v0.24.0 # installs syft
|
||||||
- name: Set up QEMU
|
- name: Set up QEMU
|
||||||
uses: docker/setup-qemu-action@v3
|
uses: docker/setup-qemu-action@v4
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@v3
|
uses: docker/setup-buildx-action@v4
|
||||||
- uses: docker/login-action@v3 # login to ghcr
|
- name: Log in to GHCR
|
||||||
|
uses: docker/login-action@v4 # login to ghcr
|
||||||
with:
|
with:
|
||||||
registry: ghcr.io
|
registry: ghcr.io
|
||||||
username: ${{ github.repository_owner }}
|
username: ${{ github.repository_owner }}
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
- uses: goreleaser/goreleaser-action@v6 # run goreleaser
|
- name: Run GoReleaser
|
||||||
|
uses: goreleaser/goreleaser-action@v7 # run goreleaser
|
||||||
with:
|
with:
|
||||||
version: nightly
|
version: nightly
|
||||||
args: release --clean --verbose
|
args: release --clean --verbose
|
||||||
env:
|
env:
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
# After GoReleaser runs, attest all the files in ./dist/checksums.txt:
|
# After GoReleaser runs, attest all the files in ./dist/checksums.txt:
|
||||||
- uses: actions/attest-build-provenance@v3
|
- name: Attest Build Provenance for Archives
|
||||||
|
uses: actions/attest-build-provenance@v4
|
||||||
with:
|
with:
|
||||||
subject-checksums: ./dist/checksums.txt
|
subject-checksums: ./dist/checksums.txt
|
||||||
# After GoReleaser runs, attest all the images in ./dist/digests.txt:
|
# After GoReleaser runs, attest all the images in ./dist/digests.txt:
|
||||||
- uses: actions/attest-build-provenance@v3
|
- name: Attest Build Provenance for Docker Images
|
||||||
|
uses: actions/attest-build-provenance@v4
|
||||||
with:
|
with:
|
||||||
subject-checksums: ./dist/digests.txt
|
subject-checksums: ./dist/digests.txt
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ jobs:
|
|||||||
name: Test
|
name: Test
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@v7
|
||||||
|
|
||||||
- name: Set up Go
|
- name: Set up Go
|
||||||
uses: actions/setup-go@v6
|
uses: actions/setup-go@v6
|
||||||
@@ -21,8 +21,8 @@ jobs:
|
|||||||
|
|
||||||
- name: Setup test environment
|
- name: Setup test environment
|
||||||
run: |
|
run: |
|
||||||
go build -tags testsetup -o test-setup ./cmd/test-setup
|
go build -tags encoder_setup -o encoder-setup ./cmd/encoder-setup
|
||||||
./test-setup
|
./encoder-setup
|
||||||
|
|
||||||
- name: Run tests
|
- name: Run tests
|
||||||
run: |
|
run: |
|
||||||
@@ -31,7 +31,7 @@ jobs:
|
|||||||
|
|
||||||
- name: Upload test result artifact
|
- name: Upload test result artifact
|
||||||
if: ${{ !cancelled() }}
|
if: ${{ !cancelled() }}
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@v7
|
||||||
with:
|
with:
|
||||||
name: test-results
|
name: test-results
|
||||||
path: |
|
path: |
|
||||||
@@ -46,6 +46,6 @@ jobs:
|
|||||||
files: test-results/junit.xml
|
files: test-results/junit.xml
|
||||||
- name: Upload coverage reports to Codecov
|
- name: Upload coverage reports to Codecov
|
||||||
if: ${{ !cancelled() }}
|
if: ${{ !cancelled() }}
|
||||||
uses: codecov/codecov-action@v5
|
uses: codecov/codecov-action@v7
|
||||||
with:
|
with:
|
||||||
token: ${{ secrets.CODECOV_TOKEN }}
|
token: ${{ secrets.CODECOV_TOKEN }}
|
||||||
|
|||||||
+81
-40
@@ -1,11 +1,34 @@
|
|||||||
# .goreleaser.yml
|
# .goreleaser.yml
|
||||||
version: 2
|
version: 2
|
||||||
project_name: CBZOptimizer
|
project_name: CBZOptimizer
|
||||||
|
|
||||||
|
# Configures the release process on GitHub
|
||||||
|
# https://goreleaser.com/customization/release/
|
||||||
release:
|
release:
|
||||||
github:
|
github:
|
||||||
owner: belphemur
|
owner: belphemur
|
||||||
name: CBZOptimizer
|
name: CBZOptimizer
|
||||||
|
include_meta: true
|
||||||
|
# draft: false # Default is false
|
||||||
|
# prerelease: auto # Default is auto
|
||||||
|
# mode: replace # Default is append
|
||||||
|
|
||||||
|
# Configures the binary archive generation
|
||||||
|
# https://goreleaser.com/customization/archive/
|
||||||
|
archives:
|
||||||
|
- ids:
|
||||||
|
- cbzoptimizer
|
||||||
|
formats: ["tar.zst"]
|
||||||
|
format_overrides:
|
||||||
|
- # Which GOOS to override the format for.
|
||||||
|
goos: windows
|
||||||
|
formats: ["zip"] # Plural form, multiple formats. Since: v2.6
|
||||||
|
|
||||||
|
# Configures the changelog generation
|
||||||
|
# https://goreleaser.com/customization/changelog/
|
||||||
changelog:
|
changelog:
|
||||||
|
use: github
|
||||||
|
format: "{{.SHA}}: {{.Message}} (@{{.AuthorUsername}})"
|
||||||
sort: asc
|
sort: asc
|
||||||
filters:
|
filters:
|
||||||
exclude:
|
exclude:
|
||||||
@@ -22,6 +45,16 @@ changelog:
|
|||||||
- title: "Performance"
|
- title: "Performance"
|
||||||
regexp: '^.*?perf(\([[:word:]]+\))??!?:.+$'
|
regexp: '^.*?perf(\([[:word:]]+\))??!?:.+$'
|
||||||
order: 2
|
order: 2
|
||||||
|
|
||||||
|
# Hooks to run before the build process starts
|
||||||
|
# https://goreleaser.com/customization/hooks/
|
||||||
|
before:
|
||||||
|
hooks:
|
||||||
|
- go mod tidy
|
||||||
|
- go generate ./...
|
||||||
|
|
||||||
|
# Configures the Go build process
|
||||||
|
# https://goreleaser.com/customization/build/
|
||||||
builds:
|
builds:
|
||||||
- id: cbzoptimizer
|
- id: cbzoptimizer
|
||||||
main: cmd/cbzoptimizer/main.go
|
main: cmd/cbzoptimizer/main.go
|
||||||
@@ -44,33 +77,61 @@ builds:
|
|||||||
- -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{ .CommitDate }}
|
- -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{ .CommitDate }}
|
||||||
env:
|
env:
|
||||||
- CGO_ENABLED=0
|
- CGO_ENABLED=0
|
||||||
# config the checksum filename
|
- id: encoder-setup
|
||||||
# https://goreleaser.com/customization/checksum
|
main: cmd/encoder-setup/main.go
|
||||||
|
binary: encoder-setup
|
||||||
|
goos:
|
||||||
|
- linux
|
||||||
|
goarch:
|
||||||
|
- amd64
|
||||||
|
- arm64
|
||||||
|
# ensures mod timestamp to be the commit timestamp
|
||||||
|
mod_timestamp: "{{ .CommitTimestamp }}"
|
||||||
|
flags:
|
||||||
|
# trims path
|
||||||
|
- -trimpath
|
||||||
|
tags:
|
||||||
|
- encoder_setup
|
||||||
|
ldflags:
|
||||||
|
- -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{ .CommitDate }}
|
||||||
|
env:
|
||||||
|
- CGO_ENABLED=0
|
||||||
|
|
||||||
|
# Configures the checksum file generation
|
||||||
|
# https://goreleaser.com/customization/checksum/
|
||||||
checksum:
|
checksum:
|
||||||
name_template: "checksums.txt"
|
name_template: "checksums.txt"
|
||||||
# Change the digests filename:
|
|
||||||
|
# Change the digests filename for attestation
|
||||||
|
# https://goreleaser.com/customization/docker_digest/
|
||||||
docker_digest:
|
docker_digest:
|
||||||
name_template: "digests.txt"
|
name_template: "digests.txt"
|
||||||
# create a source tarball
|
|
||||||
|
# Creates a source code archive (tar.gz and zip)
|
||||||
# https://goreleaser.com/customization/source/
|
# https://goreleaser.com/customization/source/
|
||||||
source:
|
source:
|
||||||
enabled: true
|
enabled: true
|
||||||
# proxies from the go mod proxy before building
|
|
||||||
# https://goreleaser.com/customization/gomod
|
# Configures Go Modules settings
|
||||||
|
# https://goreleaser.com/customization/gomod/
|
||||||
gomod:
|
gomod:
|
||||||
proxy: true
|
proxy: true
|
||||||
# creates SBOMs of all archives and the source tarball using syft
|
|
||||||
# https://goreleaser.com/customization/sbom
|
# Creates SBOMs (Software Bill of Materials)
|
||||||
|
# https://goreleaser.com/customization/sbom/
|
||||||
sboms:
|
sboms:
|
||||||
- artifacts: archive
|
- id: archive # Default ID for archive SBOMs
|
||||||
- id: source # Two different sbom configurations need two different IDs
|
artifacts: archive # Generate SBOMs for binary archives using Syft
|
||||||
artifacts: source
|
- id: source # Unique ID for source SBOM
|
||||||
# create a docker image
|
artifacts: source # Generate SBOM for the source code archive
|
||||||
# https://goreleaser.com/customization/docker
|
|
||||||
|
# Creates Docker images and pushes them to registries using Docker v2 API
|
||||||
|
# https://goreleaser.com/customization/docker/
|
||||||
dockers_v2:
|
dockers_v2:
|
||||||
- id: cbzoptimizer-image
|
- id: cbzoptimizer-image
|
||||||
ids:
|
ids:
|
||||||
- cbzoptimizer
|
- cbzoptimizer
|
||||||
|
- encoder-setup
|
||||||
platforms:
|
platforms:
|
||||||
- linux/amd64
|
- linux/amd64
|
||||||
- linux/arm64
|
- linux/arm64
|
||||||
@@ -79,6 +140,13 @@ dockers_v2:
|
|||||||
tags:
|
tags:
|
||||||
- "{{ .Version }}"
|
- "{{ .Version }}"
|
||||||
- latest
|
- latest
|
||||||
|
annotations:
|
||||||
|
"org.opencontainers.image.description": "CBZOptimizer is a Go-based tool designed to optimize CBZ (Comic Book Zip) and CBR (Comic Book RAR) files by converting images to a specified format and quality. This tool is useful for reducing the size of comic book archives while maintaining acceptable image quality."
|
||||||
|
"org.opencontainers.image.created": "{{.Date}}"
|
||||||
|
"org.opencontainers.image.name": "{{.ProjectName}}"
|
||||||
|
"org.opencontainers.image.revision": "{{.FullCommit}}"
|
||||||
|
"org.opencontainers.image.version": "{{.Version}}"
|
||||||
|
"org.opencontainers.image.source": "{{.GitURL}}"
|
||||||
labels:
|
labels:
|
||||||
"org.opencontainers.image.created": "{{.Date}}"
|
"org.opencontainers.image.created": "{{.Date}}"
|
||||||
"org.opencontainers.image.name": "{{.ProjectName}}"
|
"org.opencontainers.image.name": "{{.ProjectName}}"
|
||||||
@@ -86,30 +154,3 @@ dockers_v2:
|
|||||||
"org.opencontainers.image.version": "{{.Version}}"
|
"org.opencontainers.image.version": "{{.Version}}"
|
||||||
"org.opencontainers.image.source": "{{.GitURL}}"
|
"org.opencontainers.image.source": "{{.GitURL}}"
|
||||||
"org.opencontainers.image.description": "CBZOptimizer is a Go-based tool designed to optimize CBZ (Comic Book Zip) and CBR (Comic Book RAR) files by converting images to a specified format and quality. This tool is useful for reducing the size of comic book archives while maintaining acceptable image quality."
|
"org.opencontainers.image.description": "CBZOptimizer is a Go-based tool designed to optimize CBZ (Comic Book Zip) and CBR (Comic Book RAR) files by converting images to a specified format and quality. This tool is useful for reducing the size of comic book archives while maintaining acceptable image quality."
|
||||||
# signs the checksum file
|
|
||||||
# all files (including the sboms) are included in the checksum, so we don't need to sign each one if we don't want to
|
|
||||||
# https://goreleaser.com/customization/sign
|
|
||||||
signs:
|
|
||||||
- cmd: cosign
|
|
||||||
env:
|
|
||||||
- COSIGN_EXPERIMENTAL=1
|
|
||||||
certificate: "${artifact}.pem"
|
|
||||||
args:
|
|
||||||
- sign-blob
|
|
||||||
- "--output-certificate=${certificate}"
|
|
||||||
- "--output-signature=${signature}"
|
|
||||||
- "${artifact}"
|
|
||||||
- "--yes" # needed on cosign 2.0.0+
|
|
||||||
artifacts: checksum
|
|
||||||
output: true
|
|
||||||
# signs our docker image
|
|
||||||
# https://goreleaser.com/customization/docker_sign
|
|
||||||
docker_signs:
|
|
||||||
- cmd: cosign
|
|
||||||
env:
|
|
||||||
- COSIGN_EXPERIMENTAL=1
|
|
||||||
output: true
|
|
||||||
args:
|
|
||||||
- "sign"
|
|
||||||
- "${artifact}"
|
|
||||||
- "--yes" # needed on cosign 2.0.0+
|
|
||||||
|
|||||||
@@ -0,0 +1,23 @@
|
|||||||
|
# Agent Instructions
|
||||||
|
|
||||||
|
This repository contains AI-agent oriented project context.
|
||||||
|
|
||||||
|
## Read first
|
||||||
|
|
||||||
|
- `docs/project-overview.md` for architecture and execution flow.
|
||||||
|
- `docs/development.md` for build/test/lint commands and runtime prerequisites.
|
||||||
|
|
||||||
|
## Repository conventions
|
||||||
|
|
||||||
|
- Language: Go
|
||||||
|
- CLI framework: Cobra + Viper
|
||||||
|
- Logging: zerolog
|
||||||
|
- Error wrapping: `fmt.Errorf("context: %w", err)`
|
||||||
|
- Prefer small, focused changes.
|
||||||
|
- Commit messages: follow [Conventional Commits](https://www.conventionalcommits.org/) (e.g. `fix(watch): debounce fsnotify events`, `docs: clarify commit convention`).
|
||||||
|
|
||||||
|
## Areas to know
|
||||||
|
|
||||||
|
- Watch command: `cmd/cbzoptimizer/commands/watch_command.go`
|
||||||
|
- Optimization orchestration: `internal/utils/optimize.go`
|
||||||
|
- Converter interface/impl: `pkg/converter/`
|
||||||
+24
-21
@@ -1,37 +1,40 @@
|
|||||||
FROM debian:trixie-slim
|
FROM alpine:latest
|
||||||
LABEL authors="Belphemur"
|
LABEL authors="Belphemur"
|
||||||
ARG TARGETPLATFORM
|
ARG TARGETPLATFORM
|
||||||
ARG APP_PATH=/usr/local/bin/CBZOptimizer
|
ARG APP_PATH=/usr/local/bin/CBZOptimizer
|
||||||
ENV USER=abc
|
ENV USER=abc
|
||||||
ENV CONFIG_FOLDER=/config
|
ENV CONFIG_FOLDER=/config
|
||||||
ENV PUID=99
|
ENV PUID=99
|
||||||
ENV DEBIAN_FRONTEND=noninteractive
|
# libwebp-tools (cwebp) is installed via apk below into /usr/bin; point go-webpbin
|
||||||
|
# at it directly so it doesn't try to download a glibc prebuilt binary.
|
||||||
|
ENV VENDOR_PATH=/usr/bin
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends adduser && \
|
RUN adduser \
|
||||||
addgroup --system users && \
|
-S \
|
||||||
adduser \
|
-D \
|
||||||
--system \
|
-h "${CONFIG_FOLDER}" \
|
||||||
--home "${CONFIG_FOLDER}" \
|
-u "${PUID}" \
|
||||||
--uid "${PUID}" \
|
-G users \
|
||||||
--ingroup users \
|
-s /bin/bash \
|
||||||
--disabled-password \
|
"${USER}"
|
||||||
"${USER}" && \
|
|
||||||
apt-get purge -y --auto-remove adduser && \
|
|
||||||
rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
COPY ${TARGETPLATFORM}/CBZOptimizer ${APP_PATH}
|
COPY ${TARGETPLATFORM}/CBZOptimizer ${APP_PATH}
|
||||||
|
|
||||||
RUN apt-get update && \
|
RUN apk add --no-cache \
|
||||||
apt-get full-upgrade -y && \
|
|
||||||
apt-get install -y --no-install-recommends \
|
|
||||||
inotify-tools \
|
|
||||||
bash \
|
bash \
|
||||||
ca-certificates \
|
ca-certificates \
|
||||||
bash-completion && \
|
bash-completion \
|
||||||
rm -rf /var/lib/apt/lists/* && \
|
libwebp-tools && \
|
||||||
chmod +x ${APP_PATH} && \
|
chmod +x ${APP_PATH} && \
|
||||||
${APP_PATH} completion bash > /etc/bash_completion.d/CBZOptimizer.bash
|
${APP_PATH} completion bash > /etc/bash_completion.d/CBZOptimizer.bash && \
|
||||||
|
mkdir -p "${CONFIG_FOLDER}" && \
|
||||||
|
chown -R "${PUID}":users "${CONFIG_FOLDER}"
|
||||||
|
|
||||||
|
|
||||||
VOLUME ${CONFIG_FOLDER}
|
|
||||||
USER ${USER}
|
USER ${USER}
|
||||||
|
|
||||||
|
# Need to run as the user to have the right config folder created
|
||||||
|
RUN --mount=type=bind,source=${TARGETPLATFORM},target=/tmp/target \
|
||||||
|
/tmp/target/encoder-setup
|
||||||
|
|
||||||
ENTRYPOINT ["/usr/local/bin/CBZOptimizer"]
|
ENTRYPOINT ["/usr/local/bin/CBZOptimizer"]
|
||||||
|
|||||||
@@ -42,6 +42,22 @@ Optimize all CBZ/CBR files in a folder recursively:
|
|||||||
cbzconverter optimize [folder] --quality 85 --parallelism 2 --override --format webp --split
|
cbzconverter optimize [folder] --quality 85 --parallelism 2 --override --format webp --split
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The format flag can be specified in multiple ways:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# Using space-separated syntax
|
||||||
|
cbzconverter optimize [folder] --format webp
|
||||||
|
|
||||||
|
# Using short form with space
|
||||||
|
cbzconverter optimize [folder] -f webp
|
||||||
|
|
||||||
|
# Using equals syntax
|
||||||
|
cbzconverter optimize [folder] --format=webp
|
||||||
|
|
||||||
|
# Format is case-insensitive
|
||||||
|
cbzconverter optimize [folder] --format WEBP
|
||||||
|
```
|
||||||
|
|
||||||
With timeout to avoid hanging on problematic chapters:
|
With timeout to avoid hanging on problematic chapters:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
@@ -62,6 +78,16 @@ Watch a folder for new CBZ/CBR files and optimize them automatically:
|
|||||||
cbzconverter watch [folder] --quality 85 --override --format webp --split
|
cbzconverter watch [folder] --quality 85 --override --format webp --split
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Watch mode only reacts to filesystem events that occur *after* it starts; by default it does
|
||||||
|
not scan and optimize files that already exist in the folder when it starts. Run the
|
||||||
|
`optimize` command first if you need to process an existing library, then use `watch`
|
||||||
|
to keep it up to date going forward, or pass `--backfill` to have `watch` optimize the
|
||||||
|
existing files at startup before it begins watching for new changes. The only exception
|
||||||
|
is a directory that gets created/moved into the watched tree while watch mode is
|
||||||
|
running: since no per-file event is emitted for files already inside it, its existing
|
||||||
|
archives are always processed once when the directory is first detected, regardless of
|
||||||
|
`--backfill`.
|
||||||
|
|
||||||
Or with Docker:
|
Or with Docker:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
@@ -71,11 +97,14 @@ docker run -v /path/to/comics:/comics ghcr.io/belphemur/cbzoptimizer:latest watc
|
|||||||
### Flags
|
### Flags
|
||||||
|
|
||||||
- `--quality`, `-q`: Quality for conversion (0-100). Default is 85.
|
- `--quality`, `-q`: Quality for conversion (0-100). Default is 85.
|
||||||
- `--parallelism`, `-n`: Number of chapters to convert in parallel. Default is 2.
|
- `--parallelism`, `-n`: Number of chapters to convert in parallel. Default is 2. Regardless of this value, the total number of pages converted at the same time (i.e. concurrent `cwebp` processes) is capped to the number of CPU cores, so increasing parallelism spreads that budget across more chapters rather than multiplying resource usage.
|
||||||
- `--override`, `-o`: Override the original files. For CBZ files, overwrites the original. For CBR files, deletes the original CBR and creates a new CBZ. Default is false.
|
- `--override`, `-o`: Override the original files. For CBZ files, overwrites the original. For CBR files, deletes the original CBR and creates a new CBZ. Default is false.
|
||||||
- `--split`, `-s`: Split long pages into smaller chunks. Default is false.
|
- `--split`, `-s`: Split long pages into smaller chunks. Default is false.
|
||||||
- `--format`, `-f`: Format to convert the images to (e.g., webp). Default is webp.
|
- `--format`, `-f`: Format to convert the images to (currently supports: webp). Default is webp.
|
||||||
|
- Can be specified as: `--format webp`, `-f webp`, or `--format=webp`
|
||||||
|
- Case-insensitive: `webp`, `WEBP`, and `WebP` are all valid
|
||||||
- `--timeout`, `-t`: Maximum time allowed for converting a single chapter (e.g., 30s, 5m, 1h). 0 means no timeout. Default is 0.
|
- `--timeout`, `-t`: Maximum time allowed for converting a single chapter (e.g., 30s, 5m, 1h). 0 means no timeout. Default is 0.
|
||||||
|
- `--backfill`: *(`watch` only)* Optimize CBZ/CBR files that already exist in the watched folder at startup, before watching for new changes. Default is false.
|
||||||
- `--log`, `-l`: Set log level; can be 'panic', 'fatal', 'error', 'warn', 'info', 'debug', or 'trace'. Default is info.
|
- `--log`, `-l`: Set log level; can be 'panic', 'fatal', 'error', 'warn', 'info', 'debug', or 'trace'. Default is info.
|
||||||
|
|
||||||
## Logging
|
## Logging
|
||||||
@@ -139,15 +168,81 @@ LOG_LEVEL=warn cbzconverter optimize comics/
|
|||||||
docker run -e LOG_LEVEL=debug -v /path/to/comics:/comics ghcr.io/belphemur/cbzoptimizer:latest optimize /comics
|
docker run -e LOG_LEVEL=debug -v /path/to/comics:/comics ghcr.io/belphemur/cbzoptimizer:latest optimize /comics
|
||||||
```
|
```
|
||||||
|
|
||||||
## Requirements
|
|
||||||
|
|
||||||
- For Docker usage: No additional requirements needed
|
|
||||||
- For binary usage: Needs `libwebp` installed on the system for WebP conversion
|
|
||||||
|
|
||||||
## Docker Image
|
## Docker Image
|
||||||
|
|
||||||
The official Docker image is available at: `ghcr.io/belphemur/cbzoptimizer:latest`
|
The official Docker image is available at: `ghcr.io/belphemur/cbzoptimizer:latest`
|
||||||
|
|
||||||
|
### Docker Compose
|
||||||
|
|
||||||
|
You can use Docker Compose to run CBZOptimizer with persistent configuration. Create a `docker-compose.yml` file:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
version: '3.8'
|
||||||
|
|
||||||
|
services:
|
||||||
|
cbzoptimizer:
|
||||||
|
image: ghcr.io/belphemur/cbzoptimizer:latest
|
||||||
|
container_name: cbzoptimizer
|
||||||
|
environment:
|
||||||
|
# Set log level (panic, fatal, error, warn, info, debug, trace)
|
||||||
|
- LOG_LEVEL=info
|
||||||
|
# User and Group ID for file permissions
|
||||||
|
- PUID=99
|
||||||
|
- PGID=100
|
||||||
|
volumes:
|
||||||
|
# Mount your comics directory
|
||||||
|
- /path/to/your/comics:/comics
|
||||||
|
# Optional: Mount a config directory for persistent settings
|
||||||
|
- ./config:/config
|
||||||
|
# Example: Optimize all comics in the /comics directory
|
||||||
|
command: optimize /comics --quality 85 --parallelism 2 --override --format webp --split
|
||||||
|
restart: unless-stopped
|
||||||
|
```
|
||||||
|
|
||||||
|
For watch mode, you can create a separate service:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
cbzoptimizer-watch:
|
||||||
|
image: ghcr.io/belphemur/cbzoptimizer:latest
|
||||||
|
container_name: cbzoptimizer-watch
|
||||||
|
environment:
|
||||||
|
- LOG_LEVEL=info
|
||||||
|
- PUID=99
|
||||||
|
- PGID=100
|
||||||
|
volumes:
|
||||||
|
- /path/to/watch/directory:/watch
|
||||||
|
- ./config:/config
|
||||||
|
# Watch for new files and automatically optimize them
|
||||||
|
command: watch /watch --quality 85 --override --format webp --split
|
||||||
|
restart: unless-stopped
|
||||||
|
```
|
||||||
|
|
||||||
|
**Important Notes:**
|
||||||
|
- Replace `/path/to/your/comics` and `/path/to/watch/directory` with your actual directory paths
|
||||||
|
- The `PUID` and `PGID` environment variables control file permissions (default: 99/100)
|
||||||
|
- The `LOG_LEVEL` environment variable sets the logging verbosity
|
||||||
|
- For one-time optimization, remove the `restart: unless-stopped` line
|
||||||
|
- Watch mode only works on Linux systems
|
||||||
|
|
||||||
|
#### Running with Docker Compose
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# Start the service (one-time optimization)
|
||||||
|
docker-compose up cbzoptimizer
|
||||||
|
|
||||||
|
# Start in detached mode
|
||||||
|
docker-compose up -d cbzoptimizer
|
||||||
|
|
||||||
|
# Start watch mode service
|
||||||
|
docker-compose up -d cbzoptimizer-watch
|
||||||
|
|
||||||
|
# View logs
|
||||||
|
docker-compose logs -f cbzoptimizer
|
||||||
|
|
||||||
|
# Stop services
|
||||||
|
docker-compose down
|
||||||
|
```
|
||||||
|
|
||||||
## Troubleshooting
|
## Troubleshooting
|
||||||
|
|
||||||
If you encounter issues:
|
If you encounter issues:
|
||||||
|
|||||||
Executable
BIN
Binary file not shown.
@@ -0,0 +1,100 @@
|
|||||||
|
package commands
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/belphemur/CBZOptimizer/v2/pkg/converter/constant"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"github.com/spf13/viper"
|
||||||
|
"github.com/thediveo/enumflag/v2"
|
||||||
|
)
|
||||||
|
|
||||||
|
// setupFormatFlag sets up the format flag for a command.
|
||||||
|
//
|
||||||
|
// Parameters:
|
||||||
|
// - cmd: The Cobra command to add the format flag to
|
||||||
|
// - converterType: Pointer to the ConversionFormat variable that will store the flag value
|
||||||
|
// - bindViper: If true, binds the flag to viper for configuration file support.
|
||||||
|
// Set to true for commands that use viper for configuration (e.g., watch command),
|
||||||
|
// and false for commands that don't (e.g., optimize command).
|
||||||
|
func setupFormatFlag(cmd *cobra.Command, converterType *constant.ConversionFormat, bindViper bool) {
|
||||||
|
formatFlag := enumflag.New(converterType, "format", constant.CommandValue, enumflag.EnumCaseInsensitive)
|
||||||
|
_ = formatFlag.RegisterCompletion(cmd, "format", constant.HelpText)
|
||||||
|
|
||||||
|
cmd.Flags().VarP(
|
||||||
|
formatFlag,
|
||||||
|
"format", "f",
|
||||||
|
fmt.Sprintf("Format to convert the images to: %s", constant.ListAll()))
|
||||||
|
|
||||||
|
if bindViper {
|
||||||
|
_ = viper.BindPFlag("format", cmd.Flags().Lookup("format"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// setupQualityFlag sets up the quality flag for a command.
|
||||||
|
//
|
||||||
|
// Parameters:
|
||||||
|
// - cmd: The Cobra command to add the quality flag to
|
||||||
|
// - defaultValue: The default quality value (0-100)
|
||||||
|
// - bindViper: If true, binds the flag to viper for configuration file support
|
||||||
|
func setupQualityFlag(cmd *cobra.Command, defaultValue uint8, bindViper bool) {
|
||||||
|
cmd.Flags().Uint8P("quality", "q", defaultValue, "Quality for conversion (0-100)")
|
||||||
|
if bindViper {
|
||||||
|
_ = viper.BindPFlag("quality", cmd.Flags().Lookup("quality"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// setupOverrideFlag sets up the override flag for a command.
|
||||||
|
//
|
||||||
|
// Parameters:
|
||||||
|
// - cmd: The Cobra command to add the override flag to
|
||||||
|
// - defaultValue: The default override value
|
||||||
|
// - bindViper: If true, binds the flag to viper for configuration file support
|
||||||
|
func setupOverrideFlag(cmd *cobra.Command, defaultValue bool, bindViper bool) {
|
||||||
|
cmd.Flags().BoolP("override", "o", defaultValue, "Override the original CBZ/CBR files")
|
||||||
|
if bindViper {
|
||||||
|
_ = viper.BindPFlag("override", cmd.Flags().Lookup("override"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// setupSplitFlag sets up the split flag for a command.
|
||||||
|
//
|
||||||
|
// Parameters:
|
||||||
|
// - cmd: The Cobra command to add the split flag to
|
||||||
|
// - defaultValue: The default split value
|
||||||
|
// - bindViper: If true, binds the flag to viper for configuration file support
|
||||||
|
func setupSplitFlag(cmd *cobra.Command, defaultValue bool, bindViper bool) {
|
||||||
|
cmd.Flags().BoolP("split", "s", defaultValue, "Split long pages into smaller chunks")
|
||||||
|
if bindViper {
|
||||||
|
_ = viper.BindPFlag("split", cmd.Flags().Lookup("split"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// setupTimeoutFlag sets up the timeout flag for a command.
|
||||||
|
//
|
||||||
|
// Parameters:
|
||||||
|
// - cmd: The Cobra command to add the timeout flag to
|
||||||
|
// - bindViper: If true, binds the flag to viper for configuration file support
|
||||||
|
func setupTimeoutFlag(cmd *cobra.Command, bindViper bool) {
|
||||||
|
cmd.Flags().DurationP("timeout", "t", 0, "Maximum time allowed for converting a single chapter (e.g., 30s, 5m, 1h). 0 means no timeout")
|
||||||
|
if bindViper {
|
||||||
|
_ = viper.BindPFlag("timeout", cmd.Flags().Lookup("timeout"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// setupCommonFlags sets up all common flags for optimize and watch commands.
|
||||||
|
//
|
||||||
|
// Parameters:
|
||||||
|
// - cmd: The Cobra command to add the flags to
|
||||||
|
// - converterType: Pointer to the ConversionFormat variable that will store the format flag value
|
||||||
|
// - qualityDefault: The default quality value (0-100)
|
||||||
|
// - overrideDefault: The default override value
|
||||||
|
// - splitDefault: The default split value
|
||||||
|
// - bindViper: If true, binds all flags to viper for configuration file support
|
||||||
|
func setupCommonFlags(cmd *cobra.Command, converterType *constant.ConversionFormat, qualityDefault uint8, overrideDefault bool, splitDefault bool, bindViper bool) {
|
||||||
|
setupFormatFlag(cmd, converterType, bindViper)
|
||||||
|
setupQualityFlag(cmd, qualityDefault, bindViper)
|
||||||
|
setupOverrideFlag(cmd, overrideDefault, bindViper)
|
||||||
|
setupSplitFlag(cmd, splitDefault, bindViper)
|
||||||
|
setupTimeoutFlag(cmd, bindViper)
|
||||||
|
}
|
||||||
@@ -12,7 +12,6 @@ import (
|
|||||||
"github.com/belphemur/CBZOptimizer/v2/pkg/converter/constant"
|
"github.com/belphemur/CBZOptimizer/v2/pkg/converter/constant"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/rs/zerolog/log"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
"github.com/thediveo/enumflag/v2"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var converterType constant.ConversionFormat
|
var converterType constant.ConversionFormat
|
||||||
@@ -25,19 +24,12 @@ func init() {
|
|||||||
RunE: ConvertCbzCommand,
|
RunE: ConvertCbzCommand,
|
||||||
Args: cobra.ExactArgs(1),
|
Args: cobra.ExactArgs(1),
|
||||||
}
|
}
|
||||||
formatFlag := enumflag.New(&converterType, "format", constant.CommandValue, enumflag.EnumCaseInsensitive)
|
|
||||||
_ = formatFlag.RegisterCompletion(command, "format", constant.HelpText)
|
|
||||||
|
|
||||||
command.Flags().Uint8P("quality", "q", 85, "Quality for conversion (0-100)")
|
// Setup common flags (format, quality, override, split, timeout)
|
||||||
|
setupCommonFlags(command, &converterType, 85, false, false, false)
|
||||||
|
|
||||||
|
// Setup optimize-specific flags
|
||||||
command.Flags().IntP("parallelism", "n", 2, "Number of chapters to convert in parallel")
|
command.Flags().IntP("parallelism", "n", 2, "Number of chapters to convert in parallel")
|
||||||
command.Flags().BoolP("override", "o", false, "Override the original CBZ/CBR files")
|
|
||||||
command.Flags().BoolP("split", "s", false, "Split long pages into smaller chunks")
|
|
||||||
command.Flags().DurationP("timeout", "t", 0, "Maximum time allowed for converting a single chapter (e.g., 30s, 5m, 1h). 0 means no timeout")
|
|
||||||
command.PersistentFlags().VarP(
|
|
||||||
formatFlag,
|
|
||||||
"format", "f",
|
|
||||||
fmt.Sprintf("Format to convert the images to: %s", constant.ListAll()))
|
|
||||||
command.PersistentFlags().Lookup("format").NoOptDefVal = constant.DefaultConversion.String()
|
|
||||||
|
|
||||||
AddCommand(command)
|
AddCommand(command)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package commands
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -172,3 +173,384 @@ func TestConvertCbzCommand(t *testing.T) {
|
|||||||
// Log summary
|
// Log summary
|
||||||
t.Logf("Found %d converted files", len(convertedFiles))
|
t.Logf("Found %d converted files", len(convertedFiles))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// setupTestCommand creates a test command with all required flags for testing.
|
||||||
|
// It mocks the converter.Get function and sets up a complete command with all flags.
|
||||||
|
//
|
||||||
|
// Returns:
|
||||||
|
// - *cobra.Command: A configured command ready for testing
|
||||||
|
// - func(): A cleanup function that must be deferred to restore the original converter.Get
|
||||||
|
func setupTestCommand(t *testing.T) (*cobra.Command, func()) {
|
||||||
|
t.Helper()
|
||||||
|
// Mock the converter.Get function
|
||||||
|
originalGet := converter.Get
|
||||||
|
converter.Get = func(format constant.ConversionFormat) (converter.Converter, error) {
|
||||||
|
return &MockConverter{}, nil
|
||||||
|
}
|
||||||
|
cleanup := func() { converter.Get = originalGet }
|
||||||
|
|
||||||
|
// Set up the command
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "optimize",
|
||||||
|
}
|
||||||
|
cmd.Flags().Uint8P("quality", "q", 85, "Quality for conversion (0-100)")
|
||||||
|
cmd.Flags().IntP("parallelism", "n", 1, "Number of chapters to convert in parallel")
|
||||||
|
cmd.Flags().BoolP("override", "o", false, "Override the original CBZ/CBR files")
|
||||||
|
cmd.Flags().BoolP("split", "s", false, "Split long pages into smaller chunks")
|
||||||
|
cmd.Flags().DurationP("timeout", "t", 0, "Maximum time allowed for converting a single chapter")
|
||||||
|
|
||||||
|
// Reset converterType to default before test for consistency
|
||||||
|
converterType = constant.DefaultConversion
|
||||||
|
setupFormatFlag(cmd, &converterType, false)
|
||||||
|
|
||||||
|
return cmd, cleanup
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFormatFlagWithSpace tests that the format flag works with space-separated values
|
||||||
|
func TestFormatFlagWithSpace(t *testing.T) {
|
||||||
|
// Create a temporary directory for testing
|
||||||
|
tempDir, err := os.MkdirTemp("", "test_format_space")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to create temp directory: %v", err)
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(tempDir)
|
||||||
|
|
||||||
|
cmd, cleanup := setupTestCommand(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
// Test with space-separated format flag (--format webp)
|
||||||
|
if err := cmd.ParseFlags([]string{"--format", "webp"}); err != nil {
|
||||||
|
t.Fatalf("Failed to parse flags: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute the command
|
||||||
|
err = ConvertCbzCommand(cmd, []string{tempDir})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Command execution failed with --format webp: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify the format was set correctly
|
||||||
|
if converterType != constant.WebP {
|
||||||
|
t.Errorf("Expected format to be WebP, got %v", converterType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFormatFlagWithShortForm tests that the short form of format flag works with space-separated values
|
||||||
|
func TestFormatFlagWithShortForm(t *testing.T) {
|
||||||
|
// Create a temporary directory for testing
|
||||||
|
tempDir, err := os.MkdirTemp("", "test_format_short")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to create temp directory: %v", err)
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(tempDir)
|
||||||
|
|
||||||
|
cmd, cleanup := setupTestCommand(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
// Test with short form and space (-f webp)
|
||||||
|
if err := cmd.ParseFlags([]string{"-f", "webp"}); err != nil {
|
||||||
|
t.Fatalf("Failed to parse flags: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute the command
|
||||||
|
err = ConvertCbzCommand(cmd, []string{tempDir})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Command execution failed with -f webp: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify the format was set correctly
|
||||||
|
if converterType != constant.WebP {
|
||||||
|
t.Errorf("Expected format to be WebP, got %v", converterType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFormatFlagWithEquals tests that the format flag works with equals syntax
|
||||||
|
func TestFormatFlagWithEquals(t *testing.T) {
|
||||||
|
// Create a temporary directory for testing
|
||||||
|
tempDir, err := os.MkdirTemp("", "test_format_equals")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to create temp directory: %v", err)
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(tempDir)
|
||||||
|
|
||||||
|
cmd, cleanup := setupTestCommand(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
// Test with equals syntax (--format=webp)
|
||||||
|
if err := cmd.ParseFlags([]string{"--format=webp"}); err != nil {
|
||||||
|
t.Fatalf("Failed to parse flags: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute the command
|
||||||
|
err = ConvertCbzCommand(cmd, []string{tempDir})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Command execution failed with --format=webp: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify the format was set correctly
|
||||||
|
if converterType != constant.WebP {
|
||||||
|
t.Errorf("Expected format to be WebP, got %v", converterType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFormatFlagDefaultValue tests that the default format is used when flag is not provided
|
||||||
|
func TestFormatFlagDefaultValue(t *testing.T) {
|
||||||
|
// Create a temporary directory for testing
|
||||||
|
tempDir, err := os.MkdirTemp("", "test_format_default")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to create temp directory: %v", err)
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(tempDir)
|
||||||
|
|
||||||
|
cmd, cleanup := setupTestCommand(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
// Don't set format flag - should use default
|
||||||
|
if err := cmd.ParseFlags([]string{}); err != nil {
|
||||||
|
t.Fatalf("Failed to parse flags: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute the command
|
||||||
|
err = ConvertCbzCommand(cmd, []string{tempDir})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Command execution failed with default format: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify the default format is used
|
||||||
|
if converterType != constant.DefaultConversion {
|
||||||
|
t.Errorf("Expected format to be default (%v), got %v", constant.DefaultConversion, converterType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFormatFlagCaseInsensitive tests that the format flag is case-insensitive
|
||||||
|
func TestFormatFlagCaseInsensitive(t *testing.T) {
|
||||||
|
// Create a temporary directory for testing
|
||||||
|
tempDir, err := os.MkdirTemp("", "test_format_case")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to create temp directory: %v", err)
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(tempDir)
|
||||||
|
|
||||||
|
testCases := []string{"webp", "WEBP", "WebP", "WeBp"}
|
||||||
|
|
||||||
|
for _, formatValue := range testCases {
|
||||||
|
t.Run(formatValue, func(t *testing.T) {
|
||||||
|
cmd, cleanup := setupTestCommand(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
// Test with different case variations
|
||||||
|
if err := cmd.ParseFlags([]string{"--format", formatValue}); err != nil {
|
||||||
|
t.Fatalf("Failed to parse flags: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute the command
|
||||||
|
err = ConvertCbzCommand(cmd, []string{tempDir})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Command execution failed with format '%s': %v", formatValue, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify the format was set correctly
|
||||||
|
if converterType != constant.WebP {
|
||||||
|
t.Errorf("Expected format to be WebP for input '%s', got %v", formatValue, converterType)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestConvertCbzCommand_ManyFiles_NoDeadlock tests that processing many files in parallel
|
||||||
|
// does not cause a deadlock. This reproduces the scenario where processing
|
||||||
|
// recursive folders of CBZ files with parallelism > 1 could cause a "all goroutines are asleep - deadlock!" error.
|
||||||
|
func TestConvertCbzCommand_ManyFiles_NoDeadlock(t *testing.T) {
|
||||||
|
// Create a temporary directory for testing
|
||||||
|
tempDir, err := os.MkdirTemp("", "test_cbz_many_files")
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
defer errs.CaptureGeneric(&err, os.RemoveAll, tempDir, "failed to remove temporary directory")
|
||||||
|
|
||||||
|
// Locate the testdata directory
|
||||||
|
testdataDir := filepath.Join("../../../testdata")
|
||||||
|
if _, err := os.Stat(testdataDir); os.IsNotExist(err) {
|
||||||
|
t.Fatalf("testdata directory not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create subdirectories to simulate the recursive folder structure from the bug report
|
||||||
|
subdirs := []string{"author1/book1", "author2/book2", "author3/book3", "author4/book4"}
|
||||||
|
for _, subdir := range subdirs {
|
||||||
|
err := os.MkdirAll(filepath.Join(tempDir, subdir), 0755)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to create subdirectory: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find a sample CBZ file to copy
|
||||||
|
var sampleCBZ string
|
||||||
|
err = filepath.Walk(testdataDir, func(path string, info os.FileInfo, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !info.IsDir() && strings.HasSuffix(strings.ToLower(info.Name()), ".cbz") && !strings.Contains(info.Name(), "converted") {
|
||||||
|
sampleCBZ = path
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil || sampleCBZ == "" {
|
||||||
|
t.Fatalf("Failed to find sample CBZ file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Copy the sample file to multiple locations (simulating many files to process)
|
||||||
|
numFilesPerDir := 5
|
||||||
|
totalFiles := 0
|
||||||
|
for _, subdir := range subdirs {
|
||||||
|
for i := 0; i < numFilesPerDir; i++ {
|
||||||
|
destPath := filepath.Join(tempDir, subdir, fmt.Sprintf("Chapter_%d.cbz", i+1))
|
||||||
|
data, err := os.ReadFile(sampleCBZ)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to read sample file: %v", err)
|
||||||
|
}
|
||||||
|
err = os.WriteFile(destPath, data, 0644)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to write test file: %v", err)
|
||||||
|
}
|
||||||
|
totalFiles++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Logf("Created %d test files across %d directories", totalFiles, len(subdirs))
|
||||||
|
|
||||||
|
// Mock the converter.Get function
|
||||||
|
originalGet := converter.Get
|
||||||
|
converter.Get = func(format constant.ConversionFormat) (converter.Converter, error) {
|
||||||
|
return &MockConverter{}, nil
|
||||||
|
}
|
||||||
|
defer func() { converter.Get = originalGet }()
|
||||||
|
|
||||||
|
// Set up the command with parallelism = 2 (same as the bug report)
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "optimize",
|
||||||
|
}
|
||||||
|
cmd.Flags().Uint8P("quality", "q", 85, "Quality for conversion (0-100)")
|
||||||
|
cmd.Flags().IntP("parallelism", "n", 2, "Number of chapters to convert in parallel")
|
||||||
|
cmd.Flags().BoolP("override", "o", false, "Override the original CBZ/CBR files")
|
||||||
|
cmd.Flags().BoolP("split", "s", false, "Split long pages into smaller chunks")
|
||||||
|
cmd.Flags().DurationP("timeout", "t", 0, "Maximum time allowed for converting a single chapter")
|
||||||
|
|
||||||
|
converterType = constant.DefaultConversion
|
||||||
|
setupFormatFlag(cmd, &converterType, false)
|
||||||
|
|
||||||
|
// Run the command with a timeout to detect deadlocks
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
done <- ConvertCbzCommand(cmd, []string{tempDir})
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Command execution failed: %v", err)
|
||||||
|
}
|
||||||
|
t.Logf("Command completed successfully without deadlock")
|
||||||
|
case <-time.After(60 * time.Second):
|
||||||
|
t.Fatal("Deadlock detected: Command did not complete within 60 seconds")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify that converted files were created
|
||||||
|
var convertedCount int
|
||||||
|
err = filepath.Walk(tempDir, func(path string, info os.FileInfo, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !info.IsDir() && strings.HasSuffix(info.Name(), "_converted.cbz") {
|
||||||
|
convertedCount++
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Error counting converted files: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if convertedCount != totalFiles {
|
||||||
|
t.Errorf("Expected %d converted files, found %d", totalFiles, convertedCount)
|
||||||
|
}
|
||||||
|
t.Logf("Found %d converted files as expected", convertedCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestConvertCbzCommand_HighParallelism_NoDeadlock tests processing with high parallelism setting.
|
||||||
|
func TestConvertCbzCommand_HighParallelism_NoDeadlock(t *testing.T) {
|
||||||
|
// Create a temporary directory
|
||||||
|
tempDir, err := os.MkdirTemp("", "test_cbz_high_parallel")
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
defer errs.CaptureGeneric(&err, os.RemoveAll, tempDir, "failed to remove temporary directory")
|
||||||
|
|
||||||
|
// Locate the testdata directory
|
||||||
|
testdataDir := filepath.Join("../../../testdata")
|
||||||
|
if _, err := os.Stat(testdataDir); os.IsNotExist(err) {
|
||||||
|
t.Fatalf("testdata directory not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find and copy sample CBZ files
|
||||||
|
var sampleCBZ string
|
||||||
|
err = filepath.Walk(testdataDir, func(path string, info os.FileInfo, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !info.IsDir() && strings.HasSuffix(strings.ToLower(info.Name()), ".cbz") && !strings.Contains(info.Name(), "converted") {
|
||||||
|
sampleCBZ = path
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil || sampleCBZ == "" {
|
||||||
|
t.Fatalf("Failed to find sample CBZ file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create many test files
|
||||||
|
numFiles := 15
|
||||||
|
for i := 0; i < numFiles; i++ {
|
||||||
|
destPath := filepath.Join(tempDir, fmt.Sprintf("test_file_%d.cbz", i+1))
|
||||||
|
data, err := os.ReadFile(sampleCBZ)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to read sample file: %v", err)
|
||||||
|
}
|
||||||
|
err = os.WriteFile(destPath, data, 0644)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to write test file: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mock the converter
|
||||||
|
originalGet := converter.Get
|
||||||
|
converter.Get = func(format constant.ConversionFormat) (converter.Converter, error) {
|
||||||
|
return &MockConverter{}, nil
|
||||||
|
}
|
||||||
|
defer func() { converter.Get = originalGet }()
|
||||||
|
|
||||||
|
// Test with high parallelism (8)
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "optimize",
|
||||||
|
}
|
||||||
|
cmd.Flags().Uint8P("quality", "q", 85, "Quality for conversion (0-100)")
|
||||||
|
cmd.Flags().IntP("parallelism", "n", 8, "Number of chapters to convert in parallel")
|
||||||
|
cmd.Flags().BoolP("override", "o", false, "Override the original CBZ/CBR files")
|
||||||
|
cmd.Flags().BoolP("split", "s", false, "Split long pages into smaller chunks")
|
||||||
|
cmd.Flags().DurationP("timeout", "t", 0, "Maximum time allowed for converting a single chapter")
|
||||||
|
|
||||||
|
converterType = constant.DefaultConversion
|
||||||
|
setupFormatFlag(cmd, &converterType, false)
|
||||||
|
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
done <- ConvertCbzCommand(cmd, []string{tempDir})
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Command execution failed: %v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(60 * time.Second):
|
||||||
|
t.Fatal("Deadlock detected with high parallelism")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -59,7 +59,7 @@ func init() {
|
|||||||
ef,
|
ef,
|
||||||
"log", "l",
|
"log", "l",
|
||||||
"Set log level; can be 'panic', 'fatal', 'error', 'warn', 'info', 'debug', or 'trace'")
|
"Set log level; can be 'panic', 'fatal', 'error', 'warn', 'info', 'debug', or 'trace'")
|
||||||
ef.RegisterCompletion(rootCmd, "log", enumflag.Help[zerolog.Level]{
|
if err := ef.RegisterCompletion(rootCmd, "log", enumflag.Help[zerolog.Level]{
|
||||||
zerolog.PanicLevel: "Only log panic messages",
|
zerolog.PanicLevel: "Only log panic messages",
|
||||||
zerolog.FatalLevel: "Log fatal and panic messages",
|
zerolog.FatalLevel: "Log fatal and panic messages",
|
||||||
zerolog.ErrorLevel: "Log error, fatal, and panic messages",
|
zerolog.ErrorLevel: "Log error, fatal, and panic messages",
|
||||||
@@ -67,11 +67,17 @@ func init() {
|
|||||||
zerolog.InfoLevel: "Log info, warn, error, fatal, and panic messages",
|
zerolog.InfoLevel: "Log info, warn, error, fatal, and panic messages",
|
||||||
zerolog.DebugLevel: "Log debug, info, warn, error, fatal, and panic messages",
|
zerolog.DebugLevel: "Log debug, info, warn, error, fatal, and panic messages",
|
||||||
zerolog.TraceLevel: "Log all messages including trace",
|
zerolog.TraceLevel: "Log all messages including trace",
|
||||||
})
|
}); err != nil {
|
||||||
|
panic(fmt.Errorf("failed to register log completion: %w", err))
|
||||||
|
}
|
||||||
|
|
||||||
// Add log level environment variable support
|
// Add log level environment variable support
|
||||||
viper.BindEnv("log", "LOG_LEVEL")
|
if err := viper.BindEnv("log", "LOG_LEVEL"); err != nil {
|
||||||
viper.BindPFlag("log", rootCmd.PersistentFlags().Lookup("log"))
|
panic(fmt.Errorf("failed to bind LOG_LEVEL env: %w", err))
|
||||||
|
}
|
||||||
|
if err := viper.BindPFlag("log", rootCmd.PersistentFlags().Lookup("log")); err != nil {
|
||||||
|
panic(fmt.Errorf("failed to bind log flag: %w", err))
|
||||||
|
}
|
||||||
|
|
||||||
rootCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) {
|
rootCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) {
|
||||||
ConfigureLogging()
|
ConfigureLogging()
|
||||||
|
|||||||
@@ -1,21 +1,31 @@
|
|||||||
package commands
|
package commands
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"runtime"
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
utils2 "github.com/belphemur/CBZOptimizer/v2/internal/utils"
|
utils2 "github.com/belphemur/CBZOptimizer/v2/internal/utils"
|
||||||
"github.com/belphemur/CBZOptimizer/v2/pkg/converter"
|
"github.com/belphemur/CBZOptimizer/v2/pkg/converter"
|
||||||
"github.com/belphemur/CBZOptimizer/v2/pkg/converter/constant"
|
"github.com/belphemur/CBZOptimizer/v2/pkg/converter/constant"
|
||||||
"github.com/pablodz/inotifywaitgo/inotifywaitgo"
|
"github.com/fsnotify/fsnotify"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/rs/zerolog/log"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
"github.com/thediveo/enumflag/v2"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// debounceDelay is the quiet period we wait, per path, after the last fsnotify
|
||||||
|
// event before triggering an optimization. This coalesces the bursts of
|
||||||
|
// Write/Create/Rename events fsnotify emits while a file is still being
|
||||||
|
// written (e.g. copied or downloaded into the watched folder).
|
||||||
|
const debounceDelay = 2 * time.Second
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
if runtime.GOOS != "linux" {
|
if runtime.GOOS != "linux" {
|
||||||
return
|
return
|
||||||
@@ -27,27 +37,12 @@ func init() {
|
|||||||
RunE: WatchCommand,
|
RunE: WatchCommand,
|
||||||
Args: cobra.ExactArgs(1),
|
Args: cobra.ExactArgs(1),
|
||||||
}
|
}
|
||||||
formatFlag := enumflag.New(&converterType, "format", constant.CommandValue, enumflag.EnumCaseInsensitive)
|
|
||||||
_ = formatFlag.RegisterCompletion(command, "format", constant.HelpText)
|
|
||||||
|
|
||||||
command.Flags().Uint8P("quality", "q", 85, "Quality for conversion (0-100)")
|
// Setup common flags (format, quality, override, split, timeout) with viper binding
|
||||||
_ = viper.BindPFlag("quality", command.Flags().Lookup("quality"))
|
setupCommonFlags(command, &converterType, 85, true, false, true)
|
||||||
|
|
||||||
command.Flags().BoolP("override", "o", true, "Override the original CBZ/CBR files")
|
command.Flags().Bool("backfill", false, "Optimize CBZ/CBR files that already exist in the watched folder at startup, before watching for new changes")
|
||||||
_ = viper.BindPFlag("override", command.Flags().Lookup("override"))
|
_ = viper.BindPFlag("backfill", command.Flags().Lookup("backfill"))
|
||||||
|
|
||||||
command.Flags().BoolP("split", "s", false, "Split long pages into smaller chunks")
|
|
||||||
_ = viper.BindPFlag("split", command.Flags().Lookup("split"))
|
|
||||||
|
|
||||||
command.Flags().DurationP("timeout", "t", 0, "Maximum time allowed for converting a single chapter (e.g., 30s, 5m, 1h). 0 means no timeout")
|
|
||||||
_ = viper.BindPFlag("timeout", command.Flags().Lookup("timeout"))
|
|
||||||
|
|
||||||
command.PersistentFlags().VarP(
|
|
||||||
formatFlag,
|
|
||||||
"format", "f",
|
|
||||||
fmt.Sprintf("Format to convert the images to: %s", constant.ListAll()))
|
|
||||||
command.PersistentFlags().Lookup("format").NoOptDefVal = constant.DefaultConversion.String()
|
|
||||||
_ = viper.BindPFlag("format", command.PersistentFlags().Lookup("format"))
|
|
||||||
|
|
||||||
AddCommand(command)
|
AddCommand(command)
|
||||||
}
|
}
|
||||||
@@ -72,81 +67,317 @@ func WatchCommand(_ *cobra.Command, args []string) error {
|
|||||||
|
|
||||||
timeout := viper.GetDuration("timeout")
|
timeout := viper.GetDuration("timeout")
|
||||||
|
|
||||||
|
backfill := viper.GetBool("backfill")
|
||||||
|
|
||||||
converterType := constant.FindConversionFormat(viper.GetString("format"))
|
converterType := constant.FindConversionFormat(viper.GetString("format"))
|
||||||
chapterConverter, err := converter.Get(converterType)
|
chapterConverter, err := converter.Get(converterType)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to get chapterConverter: %v", err)
|
return fmt.Errorf("failed to get chapterConverter: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = chapterConverter.PrepareConverter()
|
err = chapterConverter.PrepareConverter()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to prepare converter: %v", err)
|
return fmt.Errorf("failed to prepare converter: %w", err)
|
||||||
}
|
}
|
||||||
log.Info().Str("path", path).Bool("override", override).Uint8("quality", quality).Str("format", converterType.String()).Bool("split", split).Msg("Watching directory")
|
log.Info().Str("path", path).Bool("override", override).Uint8("quality", quality).Str("format", converterType.String()).Bool("split", split).Bool("backfill", backfill).Msg("Watching directory")
|
||||||
|
|
||||||
events := make(chan inotifywaitgo.FileEvent)
|
watcher, err := fsnotify.NewWatcher()
|
||||||
errors := make(chan error)
|
if err != nil {
|
||||||
var wg sync.WaitGroup
|
return fmt.Errorf("failed to create file watcher: %w", err)
|
||||||
|
}
|
||||||
wg.Add(1)
|
defer func() {
|
||||||
go func() {
|
if closeErr := watcher.Close(); closeErr != nil {
|
||||||
defer wg.Done()
|
log.Error().Err(closeErr).Msg("Failed to close file watcher")
|
||||||
inotifywaitgo.WatchPath(&inotifywaitgo.Settings{
|
}
|
||||||
Dir: path,
|
|
||||||
FileEvents: events,
|
|
||||||
ErrorChan: errors,
|
|
||||||
Options: &inotifywaitgo.Options{
|
|
||||||
Recursive: true,
|
|
||||||
Events: []inotifywaitgo.EVENT{
|
|
||||||
inotifywaitgo.MOVE,
|
|
||||||
inotifywaitgo.CLOSE_WRITE,
|
|
||||||
},
|
|
||||||
Monitor: true,
|
|
||||||
},
|
|
||||||
Verbose: true,
|
|
||||||
})
|
|
||||||
}()
|
}()
|
||||||
|
|
||||||
wg.Add(1)
|
// Optimization jobs run in a small worker pool so the fsnotify event loop
|
||||||
go func() {
|
// is never blocked waiting on a conversion.
|
||||||
defer wg.Done()
|
queue := newOptimizeQueue(runtime.NumCPU(), &utils2.OptimizeOptions{
|
||||||
for event := range events {
|
|
||||||
log.Debug().Str("file", event.Filename).Interface("events", event.Events).Msg("File event")
|
|
||||||
|
|
||||||
filename := strings.ToLower(event.Filename)
|
|
||||||
if !strings.HasSuffix(filename, ".cbz") && !strings.HasSuffix(filename, ".cbr") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, e := range event.Events {
|
|
||||||
switch e {
|
|
||||||
case inotifywaitgo.CLOSE_WRITE, inotifywaitgo.MOVE:
|
|
||||||
err := utils2.Optimize(&utils2.OptimizeOptions{
|
|
||||||
ChapterConverter: chapterConverter,
|
ChapterConverter: chapterConverter,
|
||||||
Path: event.Filename,
|
|
||||||
Quality: quality,
|
Quality: quality,
|
||||||
Override: override,
|
Override: override,
|
||||||
Split: split,
|
Split: split,
|
||||||
Timeout: timeout,
|
Timeout: timeout,
|
||||||
})
|
})
|
||||||
if err != nil {
|
defer queue.Stop()
|
||||||
errors <- fmt.Errorf("error processing file %s: %w", event.Filename, err)
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
// ignored
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
wg.Add(1)
|
debouncer := newEventDebouncer(debounceDelay, queue.Enqueue)
|
||||||
go func() {
|
defer debouncer.Stop()
|
||||||
defer wg.Done()
|
|
||||||
for err := range errors {
|
// Note: existing archives already present under path when the watch
|
||||||
|
// starts are left untouched unless --backfill is set. Watch mode only
|
||||||
|
// reacts to filesystem events going forward by default; use the
|
||||||
|
// `optimize` command (or pass --backfill) to process a library's
|
||||||
|
// existing contents. Archives inside a directory that is created/moved
|
||||||
|
// into the watched tree *after* startup are always back-filled below,
|
||||||
|
// since only the directory itself generates an fsnotify event.
|
||||||
|
if err := addRecursiveWatch(watcher, path); err != nil {
|
||||||
|
return fmt.Errorf("failed to watch path %s: %w", path, err)
|
||||||
|
}
|
||||||
|
maybeBackfillExistingArchives(backfill, path, debouncer.Trigger)
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case event, ok := <-watcher.Events:
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
log.Debug().Str("file", event.Name).Str("event", event.Op.String()).Msg("File event")
|
||||||
|
|
||||||
|
if event.Has(fsnotify.Create) {
|
||||||
|
fileInfo, err := os.Stat(event.Name)
|
||||||
|
if err == nil && fileInfo.IsDir() {
|
||||||
|
if err := addRecursiveWatch(watcher, event.Name); err != nil {
|
||||||
|
log.Error().Err(err).Str("path", event.Name).Msg("Failed to watch created directory")
|
||||||
|
}
|
||||||
|
// The newly discovered directory may already contain
|
||||||
|
// archives (e.g. a folder moved/copied in); back-fill them
|
||||||
|
// since no further fsnotify event will target them.
|
||||||
|
backfillExistingArchives(event.Name, debouncer.Trigger)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !shouldProcessWatchEvent(event) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if !isComicArchive(event.Name) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
debouncer.Trigger(event.Name)
|
||||||
|
case err, ok := <-watcher.Errors:
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
log.Error().Err(err).Msg("Watch error")
|
log.Error().Err(err).Msg("Watch error")
|
||||||
}
|
}
|
||||||
}()
|
}
|
||||||
|
}
|
||||||
wg.Wait()
|
|
||||||
return nil
|
// addRecursiveWatch registers a watch on rootPath and every subdirectory
|
||||||
|
// beneath it. Directories that can't be read (e.g. permission errors) are
|
||||||
|
// logged and skipped instead of aborting the whole walk, so a single bad
|
||||||
|
// subtree doesn't prevent the rest of the folder from being watched.
|
||||||
|
func addRecursiveWatch(watcher *fsnotify.Watcher, rootPath string) error {
|
||||||
|
return filepath.WalkDir(rootPath, func(path string, entry fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
log.Warn().Err(err).Str("path", path).Msg("Skipping path while setting up watch")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !entry.IsDir() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := watcher.Add(path); err != nil {
|
||||||
|
if errors.Is(err, fs.ErrPermission) {
|
||||||
|
log.Warn().Err(err).Str("path", path).Msg("Skipping unreadable directory while setting up watch")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("failed to watch directory %s: %w", path, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// backfillExistingArchives walks rootPath for comic archives that already
|
||||||
|
// exist on disk and hands them to process. This covers the case where a
|
||||||
|
// directory (potentially already containing archives) is created/moved into
|
||||||
|
// the watched tree: only the directory itself generates an fsnotify event,
|
||||||
|
// so the files inside it would otherwise never be picked up.
|
||||||
|
func backfillExistingArchives(rootPath string, process func(path string)) {
|
||||||
|
err := filepath.WalkDir(rootPath, func(path string, entry fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
log.Warn().Err(err).Str("path", path).Msg("Skipping path while scanning for existing archives")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if entry.IsDir() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !isComicArchive(path) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
process(path)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Error().Err(err).Str("path", rootPath).Msg("Failed to scan directory for existing archives")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// maybeBackfillExistingArchives runs backfillExistingArchives against
|
||||||
|
// rootPath only when enabled is true. It exists as its own function (rather
|
||||||
|
// than inlining the `if` check at the call site) so the gating decision used
|
||||||
|
// by WatchCommand can be exercised directly in tests.
|
||||||
|
func maybeBackfillExistingArchives(enabled bool, rootPath string, process func(path string)) {
|
||||||
|
if !enabled {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
backfillExistingArchives(rootPath, process)
|
||||||
|
}
|
||||||
|
|
||||||
|
func shouldProcessWatchEvent(event fsnotify.Event) bool {
|
||||||
|
return event.Has(fsnotify.Create) || event.Has(fsnotify.Write) || event.Has(fsnotify.Rename)
|
||||||
|
}
|
||||||
|
|
||||||
|
func isComicArchive(path string) bool {
|
||||||
|
filename := strings.ToLower(path)
|
||||||
|
return strings.HasSuffix(filename, ".cbz") || strings.HasSuffix(filename, ".cbr")
|
||||||
|
}
|
||||||
|
|
||||||
|
// eventDebouncer coalesces bursts of fsnotify events targeting the same path
|
||||||
|
// into a single call to onQuiet, fired after the path has been quiet for
|
||||||
|
// delay. This prevents repeated conversions being triggered while a file is
|
||||||
|
// still being written, and avoids running Optimize against a Rename event
|
||||||
|
// that reports the *old* (already gone) file name once the quiet period lets
|
||||||
|
// us re-check the path.
|
||||||
|
type eventDebouncer struct {
|
||||||
|
delay time.Duration
|
||||||
|
onQuiet func(path string)
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
timers map[string]*debounceTimer
|
||||||
|
inFlight sync.WaitGroup
|
||||||
|
stopping bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type debounceTimer struct {
|
||||||
|
timer *time.Timer
|
||||||
|
}
|
||||||
|
|
||||||
|
func newEventDebouncer(delay time.Duration, onQuiet func(path string)) *eventDebouncer {
|
||||||
|
return &eventDebouncer{
|
||||||
|
delay: delay,
|
||||||
|
onQuiet: onQuiet,
|
||||||
|
timers: make(map[string]*debounceTimer),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Trigger (re)schedules onQuiet to run for path after the debounce delay,
|
||||||
|
// resetting any previously pending timer for the same path.
|
||||||
|
func (d *eventDebouncer) Trigger(path string) {
|
||||||
|
d.mu.Lock()
|
||||||
|
defer d.mu.Unlock()
|
||||||
|
|
||||||
|
if d.stopping {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if existing, exists := d.timers[path]; exists {
|
||||||
|
if existing.timer.Stop() {
|
||||||
|
d.inFlight.Done()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
entry := &debounceTimer{}
|
||||||
|
d.inFlight.Add(1)
|
||||||
|
entry.timer = time.AfterFunc(d.delay, func() {
|
||||||
|
defer d.inFlight.Done()
|
||||||
|
|
||||||
|
d.mu.Lock()
|
||||||
|
// Only the still-current timer for this path is allowed to clear the
|
||||||
|
// entry and fire onQuiet. If Trigger raced with this callback and
|
||||||
|
// already installed a newer timer, this stale invocation must not
|
||||||
|
// delete that newer entry or fire early.
|
||||||
|
owns := d.timers[path] == entry && !d.stopping
|
||||||
|
if owns {
|
||||||
|
delete(d.timers, path)
|
||||||
|
}
|
||||||
|
d.mu.Unlock()
|
||||||
|
if owns {
|
||||||
|
d.onQuiet(path)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
d.timers[path] = entry
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop cancels all pending timers.
|
||||||
|
func (d *eventDebouncer) Stop() {
|
||||||
|
d.mu.Lock()
|
||||||
|
d.stopping = true
|
||||||
|
for path, timer := range d.timers {
|
||||||
|
if timer.timer.Stop() {
|
||||||
|
d.inFlight.Done()
|
||||||
|
}
|
||||||
|
delete(d.timers, path)
|
||||||
|
}
|
||||||
|
d.mu.Unlock()
|
||||||
|
d.inFlight.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
|
// optimizeQueue is a small worker pool that runs Optimize jobs off of the
|
||||||
|
// fsnotify event loop, so a slow conversion never blocks event draining.
|
||||||
|
type optimizeQueue struct {
|
||||||
|
jobs chan string
|
||||||
|
wg sync.WaitGroup
|
||||||
|
mu sync.RWMutex
|
||||||
|
stopped bool
|
||||||
|
options *utils2.OptimizeOptions
|
||||||
|
optimize func(options *utils2.OptimizeOptions) error
|
||||||
|
}
|
||||||
|
|
||||||
|
func newOptimizeQueue(workerCount int, options *utils2.OptimizeOptions) *optimizeQueue {
|
||||||
|
if workerCount < 1 {
|
||||||
|
workerCount = 1
|
||||||
|
}
|
||||||
|
q := &optimizeQueue{
|
||||||
|
jobs: make(chan string, 64),
|
||||||
|
options: options,
|
||||||
|
optimize: utils2.Optimize,
|
||||||
|
}
|
||||||
|
q.wg.Add(workerCount)
|
||||||
|
for i := 0; i < workerCount; i++ {
|
||||||
|
go q.worker()
|
||||||
|
}
|
||||||
|
return q
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *optimizeQueue) worker() {
|
||||||
|
defer q.wg.Done()
|
||||||
|
for path := range q.jobs {
|
||||||
|
q.process(path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// process re-checks the path right before optimizing: fsnotify Rename events
|
||||||
|
// commonly report the *old* file name (already gone by the time we act on
|
||||||
|
// it), and Write can fire while a file is still being written elsewhere. In
|
||||||
|
// override mode, Optimize may overwrite/delete the source file, so skipping
|
||||||
|
// paths that no longer exist (or that are no longer regular files) avoids
|
||||||
|
// noisy failures.
|
||||||
|
func (q *optimizeQueue) process(path string) {
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
log.Debug().Err(err).Str("file", path).Msg("Skipping watch event: path no longer accessible")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if info.IsDir() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
options := *q.options
|
||||||
|
options.Path = path
|
||||||
|
if err := q.optimize(&options); err != nil {
|
||||||
|
log.Error().Err(err).Str("file", path).Msg("Error processing file")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enqueue submits path for processing by the worker pool.
|
||||||
|
func (q *optimizeQueue) Enqueue(path string) {
|
||||||
|
q.mu.RLock()
|
||||||
|
defer q.mu.RUnlock()
|
||||||
|
if q.stopped {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
q.jobs <- path
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop closes the job queue and waits for in-flight jobs to finish.
|
||||||
|
func (q *optimizeQueue) Stop() {
|
||||||
|
q.mu.Lock()
|
||||||
|
q.stopped = true
|
||||||
|
close(q.jobs)
|
||||||
|
q.mu.Unlock()
|
||||||
|
q.wg.Wait()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,204 @@
|
|||||||
|
package commands
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
utils2 "github.com/belphemur/CBZOptimizer/v2/internal/utils"
|
||||||
|
"github.com/fsnotify/fsnotify"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestIsComicArchive(t *testing.T) {
|
||||||
|
testCases := []struct {
|
||||||
|
name string
|
||||||
|
path string
|
||||||
|
expected bool
|
||||||
|
}{
|
||||||
|
{"cbz lowercase", "/a/b/chapter.cbz", true},
|
||||||
|
{"cbr lowercase", "/a/b/chapter.cbr", true},
|
||||||
|
{"cbz uppercase", "/a/b/chapter.CBZ", true},
|
||||||
|
{"other extension", "/a/b/chapter.zip", false},
|
||||||
|
{"no extension", "/a/b/chapter", false},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range testCases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
assert.Equal(t, tc.expected, isComicArchive(tc.path))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestShouldProcessWatchEvent(t *testing.T) {
|
||||||
|
testCases := []struct {
|
||||||
|
name string
|
||||||
|
op fsnotify.Op
|
||||||
|
expected bool
|
||||||
|
}{
|
||||||
|
{"create", fsnotify.Create, true},
|
||||||
|
{"write", fsnotify.Write, true},
|
||||||
|
{"rename", fsnotify.Rename, true},
|
||||||
|
{"remove", fsnotify.Remove, false},
|
||||||
|
{"chmod", fsnotify.Chmod, false},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range testCases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
event := fsnotify.Event{Name: "file.cbz", Op: tc.op}
|
||||||
|
assert.Equal(t, tc.expected, shouldProcessWatchEvent(event))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAddRecursiveWatchSkipsUnreadableSubdirectory(t *testing.T) {
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("permission bits are not reliably enforced on Windows")
|
||||||
|
}
|
||||||
|
|
||||||
|
root := t.TempDir()
|
||||||
|
nested := filepath.Join(root, "nested")
|
||||||
|
assert.NoError(t, os.MkdirAll(nested, 0o755))
|
||||||
|
assert.NoError(t, os.Chmod(nested, 0o000))
|
||||||
|
defer func() {
|
||||||
|
assert.NoError(t, os.Chmod(nested, 0o755))
|
||||||
|
}()
|
||||||
|
if _, err := os.ReadDir(nested); err == nil {
|
||||||
|
t.Skip("cannot make nested directory unreadable in this environment")
|
||||||
|
}
|
||||||
|
|
||||||
|
watcher, err := fsnotify.NewWatcher()
|
||||||
|
assert.NoError(t, err)
|
||||||
|
defer func() {
|
||||||
|
assert.NoError(t, watcher.Close())
|
||||||
|
}()
|
||||||
|
|
||||||
|
// A regular, fully accessible tree should be watched without error.
|
||||||
|
assert.NoError(t, addRecursiveWatch(watcher, root))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackfillExistingArchivesFindsPreExistingArchives(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
assert.NoError(t, os.WriteFile(filepath.Join(root, "chapter1.cbz"), []byte("data"), 0o644))
|
||||||
|
assert.NoError(t, os.WriteFile(filepath.Join(root, "notes.txt"), []byte("data"), 0o644))
|
||||||
|
|
||||||
|
sub := filepath.Join(root, "sub")
|
||||||
|
assert.NoError(t, os.MkdirAll(sub, 0o755))
|
||||||
|
assert.NoError(t, os.WriteFile(filepath.Join(sub, "chapter2.cbr"), []byte("data"), 0o644))
|
||||||
|
|
||||||
|
var mu sync.Mutex
|
||||||
|
var found []string
|
||||||
|
backfillExistingArchives(root, func(path string) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
found = append(found, path)
|
||||||
|
})
|
||||||
|
|
||||||
|
assert.Len(t, found, 2)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEventDebouncerCoalescesBurstsIntoSingleCall(t *testing.T) {
|
||||||
|
var calls int32
|
||||||
|
debouncer := newEventDebouncer(20*time.Millisecond, func(path string) {
|
||||||
|
atomic.AddInt32(&calls, 1)
|
||||||
|
})
|
||||||
|
defer debouncer.Stop()
|
||||||
|
|
||||||
|
// Simulate a burst of events for the same path.
|
||||||
|
for i := 0; i < 5; i++ {
|
||||||
|
debouncer.Trigger("/tmp/chapter.cbz")
|
||||||
|
}
|
||||||
|
|
||||||
|
require.Eventually(t, func() bool {
|
||||||
|
return atomic.LoadInt32(&calls) == 1
|
||||||
|
}, 2*time.Second, 10*time.Millisecond)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEventDebouncerHandlesMultiplePaths(t *testing.T) {
|
||||||
|
var mu sync.Mutex
|
||||||
|
seen := make(map[string]int)
|
||||||
|
debouncer := newEventDebouncer(10*time.Millisecond, func(path string) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
seen[path]++
|
||||||
|
})
|
||||||
|
defer debouncer.Stop()
|
||||||
|
|
||||||
|
debouncer.Trigger("/tmp/a.cbz")
|
||||||
|
debouncer.Trigger("/tmp/b.cbz")
|
||||||
|
|
||||||
|
require.Eventually(t, func() bool {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
return seen["/tmp/a.cbz"] == 1 && seen["/tmp/b.cbz"] == 1
|
||||||
|
}, 2*time.Second, 10*time.Millisecond)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOptimizeQueueSkipsMissingPath(t *testing.T) {
|
||||||
|
q := newOptimizeQueue(1, &utils2.OptimizeOptions{})
|
||||||
|
defer q.Stop()
|
||||||
|
var calls int32
|
||||||
|
done := make(chan struct{}, 1)
|
||||||
|
existing := filepath.Join(t.TempDir(), "existing.cbz")
|
||||||
|
require.NoError(t, os.WriteFile(existing, []byte("data"), 0o644))
|
||||||
|
q.optimize = func(options *utils2.OptimizeOptions) error {
|
||||||
|
if options.Path == existing {
|
||||||
|
atomic.AddInt32(&calls, 1)
|
||||||
|
select {
|
||||||
|
case done <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enqueue a path that doesn't exist; process should skip it without
|
||||||
|
// panicking or blocking since it never reaches utils2.Optimize.
|
||||||
|
q.Enqueue(filepath.Join(t.TempDir(), "missing.cbz"))
|
||||||
|
q.Enqueue(existing)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("timed out waiting for existing path to be processed")
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.EqualValues(t, 1, atomic.LoadInt32(&calls))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWatchCommandBackfillFlagDefaultsToFalse(t *testing.T) {
|
||||||
|
watchCmd, _, err := rootCmd.Find([]string{"watch"})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
flag := watchCmd.Flags().Lookup("backfill")
|
||||||
|
require.NotNil(t, flag, "watch command should register a --backfill flag")
|
||||||
|
assert.Equal(t, "false", flag.DefValue)
|
||||||
|
assert.Equal(t, "bool", flag.Value.Type())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMaybeBackfillExistingArchivesOnlyInvokedWhenRequested(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(root, "chapter1.cbz"), []byte("data"), 0o644))
|
||||||
|
|
||||||
|
runBackfill := func(enabled bool) []string {
|
||||||
|
var found []string
|
||||||
|
var mu sync.Mutex
|
||||||
|
process := func(path string) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
found = append(found, path)
|
||||||
|
}
|
||||||
|
// This is the exact same gating call WatchCommand makes based on the
|
||||||
|
// --backfill flag value.
|
||||||
|
maybeBackfillExistingArchives(enabled, root, process)
|
||||||
|
return found
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Empty(t, runBackfill(false), "no pre-existing archive should be processed when backfill is disabled")
|
||||||
|
assert.Len(t, runBackfill(true), 1, "pre-existing archives should be processed when backfill is enabled")
|
||||||
|
}
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
//go:build testsetup
|
//go:build encoder_setup
|
||||||
// +build testsetup
|
// +build encoder_setup
|
||||||
|
|
||||||
package main
|
package main
|
||||||
|
|
||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
fmt.Println("Setting up WebP encoder for tests...")
|
fmt.Println("Setting up WebP encoder ...")
|
||||||
if err := webp.PrepareEncoder(); err != nil {
|
if err := webp.PrepareEncoder(); err != nil {
|
||||||
log.Fatalf("Failed to prepare WebP encoder: %v", err)
|
log.Fatalf("Failed to prepare WebP encoder: %v", err)
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
version: '3.8'
|
||||||
|
|
||||||
|
services:
|
||||||
|
cbzoptimizer:
|
||||||
|
image: ghcr.io/belphemur/cbzoptimizer:latest
|
||||||
|
container_name: cbzoptimizer
|
||||||
|
environment:
|
||||||
|
# Set log level (panic, fatal, error, warn, info, debug, trace)
|
||||||
|
- LOG_LEVEL=info
|
||||||
|
# User and Group ID for file permissions
|
||||||
|
- PUID=99
|
||||||
|
- PGID=100
|
||||||
|
volumes:
|
||||||
|
# Mount your comics directory
|
||||||
|
- /path/to/your/comics:/comics
|
||||||
|
# Optional: Mount a config directory for persistent settings
|
||||||
|
- ./config:/config
|
||||||
|
# Example: Optimize all comics in the /comics directory
|
||||||
|
command: optimize /comics --quality 85 --parallelism 2 --override --format webp --split
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
# Example: Watch mode service
|
||||||
|
cbzoptimizer-watch:
|
||||||
|
image: ghcr.io/belphemur/cbzoptimizer:latest
|
||||||
|
container_name: cbzoptimizer-watch
|
||||||
|
environment:
|
||||||
|
- LOG_LEVEL=info
|
||||||
|
- PUID=99
|
||||||
|
- PGID=100
|
||||||
|
volumes:
|
||||||
|
- /path/to/watch/directory:/watch
|
||||||
|
- ./config:/config
|
||||||
|
# Watch for new files and automatically optimize them
|
||||||
|
command: watch /watch --quality 85 --override --format webp --split
|
||||||
|
restart: unless-stopped
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
# Development Guide
|
||||||
|
|
||||||
|
## Build
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go build -o cbzconverter ./cmd/cbzoptimizer
|
||||||
|
```
|
||||||
|
|
||||||
|
## Encoder setup (required for WebP conversion)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go build -tags encoder_setup -o encoder-setup ./cmd/encoder-setup
|
||||||
|
./encoder-setup
|
||||||
|
```
|
||||||
|
|
||||||
|
## Test
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go test ./...
|
||||||
|
```
|
||||||
|
|
||||||
|
## Lint
|
||||||
|
|
||||||
|
```bash
|
||||||
|
golangci-lint run
|
||||||
|
```
|
||||||
|
|
||||||
|
## Important behavior
|
||||||
|
|
||||||
|
- Input supports CBZ and CBR.
|
||||||
|
- Output is always CBZ.
|
||||||
|
- `--override` replaces source CBZ files and removes source CBR files after successful conversion.
|
||||||
|
- Watch mode performs recursive directory monitoring.
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
# Project Overview
|
||||||
|
|
||||||
|
CBZOptimizer is a Go CLI that optimizes comic archives (`.cbz` and `.cbr`) by converting page images to modern formats (currently WebP).
|
||||||
|
|
||||||
|
## High-level flow
|
||||||
|
|
||||||
|
1. Load chapters from archive files.
|
||||||
|
2. Decode pages and optionally split oversized pages.
|
||||||
|
3. Convert images with the selected converter.
|
||||||
|
4. Write optimized output as CBZ.
|
||||||
|
|
||||||
|
## Main components
|
||||||
|
|
||||||
|
- `cmd/cbzoptimizer`: CLI commands and flag wiring (`optimize`, `watch`).
|
||||||
|
- `internal/cbz`: archive loading and writing.
|
||||||
|
- `internal/manga`: chapter and page domain models.
|
||||||
|
- `internal/utils`: orchestration utilities (`optimize` flow and file helpers).
|
||||||
|
- `pkg/converter`: converter abstraction and format implementations.
|
||||||
|
|
||||||
|
## Watch mode
|
||||||
|
|
||||||
|
`watch` monitors a directory tree for archive file changes and runs optimization automatically. By default it only reacts to changes going forward; pass `--backfill` to also optimize archives that already exist in the folder at startup.
|
||||||
|
|
||||||
|
## Key runtime requirements
|
||||||
|
|
||||||
|
- Go 1.25+
|
||||||
|
- WebP encoder setup via `cmd/encoder-setup` for WebP conversion tests and runtime support.
|
||||||
@@ -1,21 +1,21 @@
|
|||||||
module github.com/belphemur/CBZOptimizer/v2
|
module github.com/belphemur/CBZOptimizer/v2
|
||||||
|
|
||||||
go 1.25
|
go 1.26
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
|
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
|
||||||
github.com/belphemur/go-webpbin/v2 v2.1.0
|
github.com/belphemur/go-webpbin/v2 v2.1.0
|
||||||
github.com/mholt/archives v0.1.3
|
github.com/fsnotify/fsnotify v1.10.1
|
||||||
|
github.com/mholt/archives v0.1.5
|
||||||
github.com/oliamb/cutter v0.2.2
|
github.com/oliamb/cutter v0.2.2
|
||||||
github.com/pablodz/inotifywaitgo v0.0.9
|
github.com/rs/zerolog v1.35.1
|
||||||
github.com/rs/zerolog v1.34.0
|
github.com/samber/lo v1.53.0
|
||||||
github.com/samber/lo v1.51.0
|
github.com/spf13/cobra v1.10.2
|
||||||
github.com/spf13/cobra v1.10.1
|
|
||||||
github.com/spf13/viper v1.21.0
|
github.com/spf13/viper v1.21.0
|
||||||
github.com/stretchr/testify v1.11.1
|
github.com/stretchr/testify v1.11.1
|
||||||
github.com/thediveo/enumflag/v2 v2.0.7
|
github.com/thediveo/enumflag/v2 v2.2.1
|
||||||
golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b
|
golang.org/x/exp v0.0.0-20260611194520-c48552f49976
|
||||||
golang.org/x/image v0.31.0
|
golang.org/x/image v0.43.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
@@ -27,7 +27,6 @@ require (
|
|||||||
github.com/bodgit/windows v1.0.1 // indirect
|
github.com/bodgit/windows v1.0.1 // indirect
|
||||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||||
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
|
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
|
||||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||||
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
||||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||||
@@ -37,7 +36,7 @@ require (
|
|||||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||||
github.com/mikelolasagasti/xz v1.0.1 // indirect
|
github.com/mikelolasagasti/xz v1.0.1 // indirect
|
||||||
github.com/minio/minlz v1.0.1 // indirect
|
github.com/minio/minlz v1.0.1 // indirect
|
||||||
github.com/nwaples/rardecode/v2 v2.1.1 // indirect
|
github.com/nwaples/rardecode/v2 v2.2.0 // indirect
|
||||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||||
github.com/pierrec/lz4/v4 v4.1.22 // indirect
|
github.com/pierrec/lz4/v4 v4.1.22 // indirect
|
||||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||||
@@ -51,7 +50,7 @@ require (
|
|||||||
github.com/ulikunitz/xz v0.5.15 // indirect
|
github.com/ulikunitz/xz v0.5.15 // indirect
|
||||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||||
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
|
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
|
||||||
golang.org/x/sys v0.36.0 // indirect
|
golang.org/x/sys v0.45.0 // indirect
|
||||||
golang.org/x/text v0.29.0 // indirect
|
golang.org/x/text v0.38.0 // indirect
|
||||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -17,6 +17,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
|
|||||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||||
|
github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
|
||||||
|
github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
|
||||||
github.com/STARRY-S/zip v0.2.3 h1:luE4dMvRPDOWQdeDdUxUoZkzUIpTccdKdhHHsQJ1fm4=
|
github.com/STARRY-S/zip v0.2.3 h1:luE4dMvRPDOWQdeDdUxUoZkzUIpTccdKdhHHsQJ1fm4=
|
||||||
github.com/STARRY-S/zip v0.2.3/go.mod h1:lqJ9JdeRipyOQJrYSOtpNAiaesFO6zVDsE8GIGFaoSk=
|
github.com/STARRY-S/zip v0.2.3/go.mod h1:lqJ9JdeRipyOQJrYSOtpNAiaesFO6zVDsE8GIGFaoSk=
|
||||||
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
|
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
|
||||||
@@ -38,7 +40,6 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
|
|||||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||||
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
@@ -51,18 +52,16 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m
|
|||||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
|
||||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
|
||||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||||
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
|
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||||
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
|
|
||||||
github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
|
github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
|
||||||
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
|
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
|
||||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||||
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||||
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||||
@@ -88,8 +87,8 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi
|
|||||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||||
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||||
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||||
github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg=
|
github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg=
|
||||||
github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
|
github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI=
|
||||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||||
@@ -116,35 +115,29 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
|||||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
|
||||||
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
|
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
|
||||||
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
|
||||||
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
|
||||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||||
github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
|
github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
|
||||||
github.com/mholt/archives v0.1.3 h1:aEAaOtNra78G+TvV5ohmXrJOAzf++dIlYeDW3N9q458=
|
github.com/mholt/archives v0.1.5 h1:Fh2hl1j7VEhc6DZs2DLMgiBNChUux154a1G+2esNvzQ=
|
||||||
github.com/mholt/archives v0.1.3/go.mod h1:LUCGp++/IbV/I0Xq4SzcIR6uwgeh2yjnQWamjRQfLTU=
|
github.com/mholt/archives v0.1.5/go.mod h1:3TPMmBLPsgszL+1As5zECTuKwKvIfj6YcwWPpeTAXF4=
|
||||||
github.com/mikelolasagasti/xz v1.0.1 h1:Q2F2jX0RYJUG3+WsM+FJknv+6eVjsjXNDV0KJXZzkD0=
|
github.com/mikelolasagasti/xz v1.0.1 h1:Q2F2jX0RYJUG3+WsM+FJknv+6eVjsjXNDV0KJXZzkD0=
|
||||||
github.com/mikelolasagasti/xz v1.0.1/go.mod h1:muAirjiOUxPRXwm9HdDtB3uoRPrGnL85XHtokL9Hcgc=
|
github.com/mikelolasagasti/xz v1.0.1/go.mod h1:muAirjiOUxPRXwm9HdDtB3uoRPrGnL85XHtokL9Hcgc=
|
||||||
github.com/minio/minlz v1.0.1 h1:OUZUzXcib8diiX+JYxyRLIdomyZYzHct6EShOKtQY2A=
|
github.com/minio/minlz v1.0.1 h1:OUZUzXcib8diiX+JYxyRLIdomyZYzHct6EShOKtQY2A=
|
||||||
github.com/minio/minlz v1.0.1/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec=
|
github.com/minio/minlz v1.0.1/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec=
|
||||||
github.com/nwaples/rardecode/v2 v2.1.1 h1:OJaYalXdliBUXPmC8CZGQ7oZDxzX1/5mQmgn0/GASew=
|
github.com/nwaples/rardecode/v2 v2.2.0 h1:4ufPGHiNe1rYJxYfehALLjup4Ls3ck42CWwjKiOqu0A=
|
||||||
github.com/nwaples/rardecode/v2 v2.1.1/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw=
|
github.com/nwaples/rardecode/v2 v2.2.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw=
|
||||||
github.com/oliamb/cutter v0.2.2 h1:Lfwkya0HHNU1YLnGv2hTkzHfasrSMkgv4Dn+5rmlk3k=
|
github.com/oliamb/cutter v0.2.2 h1:Lfwkya0HHNU1YLnGv2hTkzHfasrSMkgv4Dn+5rmlk3k=
|
||||||
github.com/oliamb/cutter v0.2.2/go.mod h1:4BenG2/4GuRBDbVm/OPahDVqbrOemzpPiG5mi1iryBU=
|
github.com/oliamb/cutter v0.2.2/go.mod h1:4BenG2/4GuRBDbVm/OPahDVqbrOemzpPiG5mi1iryBU=
|
||||||
github.com/onsi/ginkgo/v2 v2.22.2 h1:/3X8Panh8/WwhU/3Ssa6rCKqPLuAkVY2I0RoyDLySlU=
|
github.com/onsi/ginkgo/v2 v2.29.0 h1:rfh+ZFjgJhYWRoIqVf3Uwx/W20yLrcrE2h2GmYVRaag=
|
||||||
github.com/onsi/ginkgo/v2 v2.22.2/go.mod h1:oeMosUL+8LtarXBHu/c0bx2D/K9zyQ6uX3cTyztHwsk=
|
github.com/onsi/ginkgo/v2 v2.29.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44=
|
||||||
github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8=
|
github.com/onsi/gomega v1.41.0 h1:OwKp4pXNgVxf6sCplzYo794OFNuoL2q2SBMU5NSWOjA=
|
||||||
github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY=
|
github.com/onsi/gomega v1.41.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A=
|
||||||
github.com/pablodz/inotifywaitgo v0.0.9 h1:njquRbBU7fuwIe5rEvtaniVBjwWzcpdUVptSgzFqZsw=
|
|
||||||
github.com/pablodz/inotifywaitgo v0.0.9/go.mod h1:hAfx2oN+WKg8miwUKPs52trySpPignlRBRxWcXVHku0=
|
|
||||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||||
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
|
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
|
||||||
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
@@ -153,18 +146,15 @@ github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ
|
|||||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||||
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
|
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
|
||||||
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
|
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
|
||||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI=
|
||||||
github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY=
|
github.com/rs/zerolog v1.35.1/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw=
|
||||||
github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
|
|
||||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
|
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
|
||||||
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
||||||
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
|
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
|
||||||
github.com/samber/lo v1.51.0 h1:kysRYLbHy/MB7kQZf5DSN50JHmMsNEdeY24VzJFu7wI=
|
github.com/samber/lo v1.53.0 h1:t975lj2py4kJPQ6haz1QMgtId2gtmfktACxIXArw3HM=
|
||||||
github.com/samber/lo v1.51.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0=
|
github.com/samber/lo v1.53.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0=
|
||||||
github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg=
|
github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg=
|
||||||
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
|
|
||||||
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
|
||||||
github.com/sorairolake/lzip-go v0.3.8 h1:j5Q2313INdTA80ureWYRhX+1K78mUXfMoPZCw/ivWik=
|
github.com/sorairolake/lzip-go v0.3.8 h1:j5Q2313INdTA80ureWYRhX+1K78mUXfMoPZCw/ivWik=
|
||||||
github.com/sorairolake/lzip-go v0.3.8/go.mod h1:JcBqGMV0frlxwrsE9sMWXDjqn3EeVf0/54YPsw66qkU=
|
github.com/sorairolake/lzip-go v0.3.8/go.mod h1:JcBqGMV0frlxwrsE9sMWXDjqn3EeVf0/54YPsw66qkU=
|
||||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
|
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
|
||||||
@@ -173,8 +163,8 @@ github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
|
|||||||
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
|
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
|
||||||
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
||||||
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
||||||
github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
|
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
||||||
github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
|
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
||||||
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||||
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||||
@@ -194,10 +184,10 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
|
|||||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||||
github.com/thediveo/enumflag/v2 v2.0.7 h1:uxXDU+rTel7Hg4X0xdqICpG9rzuI/mzLAEYXWLflOfs=
|
github.com/thediveo/enumflag/v2 v2.2.1 h1:sB6zJBA7G5Qk5tcCK5f25H8Rfern7idwTASdBxY4inQ=
|
||||||
github.com/thediveo/enumflag/v2 v2.0.7/go.mod h1:bWlnNvTJuUK+huyzf3WECFLy557Ttlc+yk3o+BPs0EA=
|
github.com/thediveo/enumflag/v2 v2.2.1/go.mod h1:Fa35DiSMi7oIXNc1VJPktJVlsk4NPW8dYY3Zjvhx+S4=
|
||||||
github.com/thediveo/success v1.0.2 h1:w+r3RbSjLmd7oiNnlCblfGqItcsaShcuAorRVh/+0xk=
|
github.com/thediveo/success v1.3.1 h1:SQ/ICN55yYxyEpgh0XGQwG+A4mMLGa2MYp+fxnIKoYs=
|
||||||
github.com/thediveo/success v1.0.2/go.mod h1:hdPJB77k70w764lh8uLUZgNhgeTl3DYeZ4d4bwMO2CU=
|
github.com/thediveo/success v1.3.1/go.mod h1:Wlj+S4i3x4pLZEO/OY/cEDfpChUoxw02BRYjcGjH+Zw=
|
||||||
github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||||
github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
|
github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
|
||||||
github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||||
@@ -225,12 +215,12 @@ golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE
|
|||||||
golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||||
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||||
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
|
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
|
||||||
golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b h1:DXr+pvt3nC887026GRP39Ej11UATqWDmWuS99x26cD0=
|
golang.org/x/exp v0.0.0-20260611194520-c48552f49976 h1:X8Hz2ImujgbmetVuW+w2YkyZChE3cBpZi2P158rTG9M=
|
||||||
golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b/go.mod h1:4QTo5u+SEIbbKW1RacMZq1YEfOBqeXa19JeshGi+zc4=
|
golang.org/x/exp v0.0.0-20260611194520-c48552f49976/go.mod h1:vnf4pv9iKZXY58sQE1L86zmNWJ4159e1RkcWiLCkeEY=
|
||||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||||
golang.org/x/image v0.31.0 h1:mLChjE2MV6g1S7oqbXC0/UcKijjm5fnJLUYKIYrLESA=
|
golang.org/x/image v0.43.0 h1:FLxcP4ec2350nTfOC8ysKtqYSIFbk/QGjw1ZHNP4tsY=
|
||||||
golang.org/x/image v0.31.0/go.mod h1:R9ec5Lcp96v9FTF+ajwaH3uGxPH4fKfHHAVbUILxghA=
|
golang.org/x/image v0.43.0/go.mod h1:rrpelvGFt+kLPAjPM4HeWPgrl0FtafueU//e5N0qk/Q=
|
||||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||||
@@ -247,6 +237,8 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
|
|||||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||||
|
golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
|
||||||
|
golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
|
||||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
@@ -263,8 +255,8 @@ golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLL
|
|||||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||||
golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
|
golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
|
||||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
|
||||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||||
@@ -277,8 +269,8 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ
|
|||||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
|
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
|
||||||
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
@@ -295,12 +287,10 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
|
||||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
|
||||||
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
|
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||||
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
|
||||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||||
@@ -311,8 +301,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
|||||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||||
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
|
golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
|
||||||
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
|
golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
|
||||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
@@ -340,8 +330,8 @@ golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapK
|
|||||||
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||||
golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||||
golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=
|
golang.org/x/tools v0.46.0 h1:7jTurBkPZu4moS/Uy4OQT1M+QBlsj3wejyZwsT8Z7rk=
|
||||||
golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
|
golang.org/x/tools v0.46.0/go.mod h1:FrD85F8l+NWL+9XWBSyVSHO6Ne4jutsfIFba7AWQ5Ys=
|
||||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package cbz
|
|||||||
import (
|
import (
|
||||||
"archive/zip"
|
"archive/zip"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -56,7 +57,7 @@ func WriteChapterToCBZ(chapter *manga.Chapter, outputFilePath string) error {
|
|||||||
Bool("is_splitted", page.IsSplitted).
|
Bool("is_splitted", page.IsSplitted).
|
||||||
Uint16("split_part", page.SplitPartIndex).
|
Uint16("split_part", page.SplitPartIndex).
|
||||||
Str("filename", fileName).
|
Str("filename", fileName).
|
||||||
Int("size", len(page.Contents.Bytes())).
|
Uint64("size", page.Size).
|
||||||
Msg("Writing page to CBZ archive")
|
Msg("Writing page to CBZ archive")
|
||||||
|
|
||||||
// Create a new file in the ZIP archive
|
// Create a new file in the ZIP archive
|
||||||
@@ -70,17 +71,30 @@ func WriteChapterToCBZ(chapter *manga.Chapter, outputFilePath string) error {
|
|||||||
return fmt.Errorf("failed to create file in .cbz: %w", err)
|
return fmt.Errorf("failed to create file in .cbz: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write the page contents to the file
|
// Stream the page contents into the archive. This transparently
|
||||||
bytesWritten, err := fileWriter.Write(page.Contents.Bytes())
|
// handles pages staged on disk (see manga.Page.TempFilePath) so we
|
||||||
|
// never need to hold the whole chapter's contents in memory at once.
|
||||||
|
pageReader, err := page.Open()
|
||||||
|
if err != nil {
|
||||||
|
log.Error().Str("output_path", outputFilePath).Str("filename", fileName).Err(err).Msg("Failed to open page contents")
|
||||||
|
return fmt.Errorf("failed to open page contents: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
bytesWritten, err := io.Copy(fileWriter, pageReader)
|
||||||
|
closeErr := pageReader.Close()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Str("output_path", outputFilePath).Str("filename", fileName).Err(err).Msg("Failed to write page contents")
|
log.Error().Str("output_path", outputFilePath).Str("filename", fileName).Err(err).Msg("Failed to write page contents")
|
||||||
return fmt.Errorf("failed to write page contents: %w", err)
|
return fmt.Errorf("failed to write page contents: %w", err)
|
||||||
}
|
}
|
||||||
|
if closeErr != nil {
|
||||||
|
log.Error().Str("output_path", outputFilePath).Str("filename", fileName).Err(closeErr).Msg("Failed to close page contents reader")
|
||||||
|
return fmt.Errorf("failed to close page contents reader: %w", closeErr)
|
||||||
|
}
|
||||||
|
|
||||||
log.Debug().
|
log.Debug().
|
||||||
Str("output_path", outputFilePath).
|
Str("output_path", outputFilePath).
|
||||||
Str("filename", fileName).
|
Str("filename", fileName).
|
||||||
Int("bytes_written", bytesWritten).
|
Int64("bytes_written", bytesWritten).
|
||||||
Msg("Page written successfully")
|
Msg("Page written successfully")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,10 @@
|
|||||||
package manga
|
package manga
|
||||||
|
|
||||||
import "time"
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
type Chapter struct {
|
type Chapter struct {
|
||||||
// FilePath is the path to the chapter's directory.
|
// FilePath is the path to the chapter's directory.
|
||||||
@@ -13,6 +17,11 @@ type Chapter struct {
|
|||||||
IsConverted bool
|
IsConverted bool
|
||||||
// ConvertedTime is a pointer to a time.Time object that indicates when the chapter was converted. Nil mean not converted.
|
// ConvertedTime is a pointer to a time.Time object that indicates when the chapter was converted. Nil mean not converted.
|
||||||
ConvertedTime time.Time
|
ConvertedTime time.Time
|
||||||
|
// TempDir, when non-empty, is a staging temp folder holding converted
|
||||||
|
// page contents on disk (see Page.TempFilePath) rather than fully in
|
||||||
|
// memory. It should be removed via Cleanup once the chapter has been
|
||||||
|
// written out (or is no longer needed).
|
||||||
|
TempDir string
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetConverted sets the IsConverted field to true and sets the ConvertedTime field to the current time.
|
// SetConverted sets the IsConverted field to true and sets the ConvertedTime field to the current time.
|
||||||
@@ -20,3 +29,16 @@ func (chapter *Chapter) SetConverted() {
|
|||||||
chapter.IsConverted = true
|
chapter.IsConverted = true
|
||||||
chapter.ConvertedTime = time.Now()
|
chapter.ConvertedTime = time.Now()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Cleanup removes the chapter's staging temp folder (if any), releasing any
|
||||||
|
// page contents that were staged to disk during conversion.
|
||||||
|
func (chapter *Chapter) Cleanup() error {
|
||||||
|
if chapter.TempDir == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := os.RemoveAll(chapter.TempDir); err != nil {
|
||||||
|
return fmt.Errorf("failed to remove staging temp folder: %w", err)
|
||||||
|
}
|
||||||
|
chapter.TempDir = ""
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
+70
-2
@@ -1,6 +1,13 @@
|
|||||||
package manga
|
package manga
|
||||||
|
|
||||||
import "bytes"
|
import (
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/rs/zerolog/log"
|
||||||
|
)
|
||||||
|
|
||||||
type Page struct {
|
type Page struct {
|
||||||
// Index of the page in the chapter.
|
// Index of the page in the chapter.
|
||||||
@@ -9,10 +16,71 @@ type Page struct {
|
|||||||
Extension string `json:"extension" jsonschema:"description=Extension of the page image."`
|
Extension string `json:"extension" jsonschema:"description=Extension of the page image."`
|
||||||
// Size of the page in bytes
|
// Size of the page in bytes
|
||||||
Size uint64 `json:"-"`
|
Size uint64 `json:"-"`
|
||||||
// Contents of the page
|
// Contents of the page. Nil when the page contents have been staged to
|
||||||
|
// disk (see TempFilePath) to bound memory usage.
|
||||||
Contents *bytes.Buffer `json:"-"`
|
Contents *bytes.Buffer `json:"-"`
|
||||||
|
// TempFilePath, when non-empty, points to a file on disk (in a staging
|
||||||
|
// temp folder) holding the page contents instead of keeping them fully
|
||||||
|
// in memory. Use Open() to transparently read the page contents
|
||||||
|
// regardless of where they are stored.
|
||||||
|
TempFilePath string `json:"-"`
|
||||||
// IsSplitted tell us if the page was cropped to multiple pieces
|
// IsSplitted tell us if the page was cropped to multiple pieces
|
||||||
IsSplitted bool `json:"is_cropped" jsonschema:"description=Was this page cropped."`
|
IsSplitted bool `json:"is_cropped" jsonschema:"description=Was this page cropped."`
|
||||||
// SplitPartIndex represent the index of the crop if the page was cropped
|
// SplitPartIndex represent the index of the crop if the page was cropped
|
||||||
SplitPartIndex uint16 `json:"crop_part_index" jsonschema:"description=Index of the crop if the image was cropped."`
|
SplitPartIndex uint16 `json:"crop_part_index" jsonschema:"description=Index of the crop if the image was cropped."`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Open returns a reader for the page contents, transparently handling
|
||||||
|
// whether the contents are held in memory (Contents) or staged on disk
|
||||||
|
// (TempFilePath). The caller is responsible for closing the returned
|
||||||
|
// reader.
|
||||||
|
func (page *Page) Open() (io.ReadCloser, error) {
|
||||||
|
if page.TempFilePath != "" {
|
||||||
|
file, err := os.Open(page.TempFilePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to open staged page contents: %w", err)
|
||||||
|
}
|
||||||
|
return file, nil
|
||||||
|
}
|
||||||
|
if page.Contents == nil {
|
||||||
|
return nil, fmt.Errorf("page has no contents: neither TempFilePath nor Contents is set")
|
||||||
|
}
|
||||||
|
return io.NopCloser(bytes.NewReader(page.Contents.Bytes())), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stage writes the given content to a file in tempDir instead of keeping it
|
||||||
|
// in memory, updating Extension, Size and TempFilePath accordingly and
|
||||||
|
// clearing Contents. This is used after converting a page so that only the
|
||||||
|
// pages currently being processed are held in memory, bounding memory usage
|
||||||
|
// for chapters with many/large pages.
|
||||||
|
func (page *Page) Stage(tempDir string, content *bytes.Buffer, extension string) error {
|
||||||
|
file, err := os.CreateTemp(tempDir, "page-*.tmp")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create staging file: %w", err)
|
||||||
|
}
|
||||||
|
fileName := file.Name()
|
||||||
|
|
||||||
|
written, writeErr := file.Write(content.Bytes())
|
||||||
|
closeErr := file.Close()
|
||||||
|
if writeErr == nil && closeErr == nil && written == content.Len() {
|
||||||
|
page.TempFilePath = fileName
|
||||||
|
page.Extension = extension
|
||||||
|
page.Size = uint64(written)
|
||||||
|
page.Contents = nil
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Something went wrong: remove the incomplete/partial staging file
|
||||||
|
// rather than leaving corrupted data behind on disk.
|
||||||
|
if removeErr := os.Remove(fileName); removeErr != nil {
|
||||||
|
log.Warn().Str("file", fileName).Err(removeErr).Msg("Failed to remove incomplete staging file")
|
||||||
|
}
|
||||||
|
|
||||||
|
if writeErr != nil {
|
||||||
|
return fmt.Errorf("failed to write staging file: %w", writeErr)
|
||||||
|
}
|
||||||
|
if closeErr != nil {
|
||||||
|
return fmt.Errorf("failed to close staging file: %w", closeErr)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("short write to staging file: wrote %d of %d bytes", written, content.Len())
|
||||||
|
}
|
||||||
|
|||||||
@@ -91,6 +91,15 @@ func Optimize(options *OptimizeOptions) error {
|
|||||||
return fmt.Errorf("failed to convert chapter")
|
return fmt.Errorf("failed to convert chapter")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Clean up the staging temp folder (if any) used to hold converted page
|
||||||
|
// contents on disk once we're done with the chapter, regardless of
|
||||||
|
// success or failure below.
|
||||||
|
defer func() {
|
||||||
|
if cleanupErr := convertedChapter.Cleanup(); cleanupErr != nil {
|
||||||
|
log.Warn().Str("file", chapter.FilePath).Err(cleanupErr).Msg("Failed to remove staging temp folder")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
log.Debug().
|
log.Debug().
|
||||||
Str("file", chapter.FilePath).
|
Str("file", chapter.FilePath).
|
||||||
Int("original_pages", len(chapter.Pages)).
|
Int("original_pages", len(chapter.Pages)).
|
||||||
|
|||||||
@@ -51,6 +51,13 @@ func TestOptimizeIntegration(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// Skip the "large" fixtures directory: it holds the Git LFS-tracked
|
||||||
|
// fixture used exclusively by TestOptimizeIntegration_LargeFile,
|
||||||
|
// which may only be a small LFS pointer file if the content wasn't
|
||||||
|
// fetched, and shouldn't be exercised by this generic test.
|
||||||
|
if info.IsDir() && filepath.Base(path) == "large" {
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
if !info.IsDir() {
|
if !info.IsDir() {
|
||||||
fileName := strings.ToLower(info.Name())
|
fileName := strings.ToLower(info.Name())
|
||||||
if (strings.HasSuffix(fileName, ".cbz") || strings.HasSuffix(fileName, ".cbr")) && !strings.Contains(fileName, "converted") {
|
if (strings.HasSuffix(fileName, ".cbz") || strings.HasSuffix(fileName, ".cbr")) && !strings.Contains(fileName, "converted") {
|
||||||
|
|||||||
@@ -0,0 +1,121 @@
|
|||||||
|
package utils
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/belphemur/CBZOptimizer/v2/internal/utils/errs"
|
||||||
|
"github.com/belphemur/CBZOptimizer/v2/pkg/converter"
|
||||||
|
"github.com/belphemur/CBZOptimizer/v2/pkg/converter/constant"
|
||||||
|
"github.com/rs/zerolog/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
// largeTestFile is a ~1GB synthetic CBZ fixture stored via Git LFS (see
|
||||||
|
// .gitattributes). It is used to exercise the optimize pipeline with a
|
||||||
|
// chapter large enough to make in-memory-only handling impractical, and to
|
||||||
|
// validate that converted pages are streamed to/from a staging temp folder
|
||||||
|
// (see manga.Page.TempFilePath / manga.Chapter.TempDir) instead of blowing
|
||||||
|
// up memory usage.
|
||||||
|
const largeTestFile = "../../testdata/large/large_chapter.cbz"
|
||||||
|
|
||||||
|
// TestOptimizeIntegration_LargeFile is opt-in (set CBZ_RUN_LARGE_FILE_TEST=1)
|
||||||
|
// since it processes a ~1GB fixture and can take a while to run. It is
|
||||||
|
// automatically skipped if the fixture is unavailable (e.g. Git LFS content
|
||||||
|
// wasn't fetched, leaving only a pointer file) or in short mode.
|
||||||
|
func TestOptimizeIntegration_LargeFile(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("Skipping large file integration test in short mode")
|
||||||
|
}
|
||||||
|
if os.Getenv("CBZ_RUN_LARGE_FILE_TEST") == "" {
|
||||||
|
t.Skip("Skipping large file integration test; set CBZ_RUN_LARGE_FILE_TEST=1 to run it")
|
||||||
|
}
|
||||||
|
|
||||||
|
info, err := os.Stat(largeTestFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Skipf("large test fixture not found: %v", err)
|
||||||
|
}
|
||||||
|
// If Git LFS content wasn't fetched (e.g. `actions/checkout` without
|
||||||
|
// `lfs: true`), the file on disk is just a small pointer text file
|
||||||
|
// rather than the real ~1GB fixture. Detect and skip gracefully instead
|
||||||
|
// of failing the whole suite.
|
||||||
|
const minExpectedSize = 500 * 1024 * 1024 // 500MB
|
||||||
|
if info.Size() < minExpectedSize {
|
||||||
|
t.Skipf("large test fixture looks like a Git LFS pointer (size=%d), skipping; run `git lfs pull`", info.Size())
|
||||||
|
}
|
||||||
|
|
||||||
|
tempDir, err := os.MkdirTemp("", "test_optimize_large_file")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer errs.CaptureGeneric(&err, os.RemoveAll, tempDir, "failed to remove temporary directory")
|
||||||
|
|
||||||
|
converterInstance, err := converter.Get(constant.WebP)
|
||||||
|
if err != nil {
|
||||||
|
t.Skip("WebP converter not available, skipping large file integration test")
|
||||||
|
}
|
||||||
|
if err := converterInstance.PrepareConverter(); err != nil {
|
||||||
|
t.Skip("Failed to prepare WebP converter, skipping large file integration test")
|
||||||
|
}
|
||||||
|
|
||||||
|
cbzFile := filepath.Join(tempDir, "large_chapter.cbz")
|
||||||
|
if err := copyFile(largeTestFile, cbzFile); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var memBefore, memAfter runtime.MemStats
|
||||||
|
runtime.GC()
|
||||||
|
runtime.ReadMemStats(&memBefore)
|
||||||
|
|
||||||
|
options := &OptimizeOptions{
|
||||||
|
ChapterConverter: converterInstance,
|
||||||
|
Path: cbzFile,
|
||||||
|
Quality: 85,
|
||||||
|
Override: false,
|
||||||
|
Split: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
err = Optimize(options)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to optimize large chapter: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
runtime.GC()
|
||||||
|
runtime.ReadMemStats(&memAfter)
|
||||||
|
log.Info().
|
||||||
|
Uint64("heap_alloc_before", memBefore.HeapAlloc).
|
||||||
|
Uint64("heap_alloc_after", memAfter.HeapAlloc).
|
||||||
|
Int64("input_size", info.Size()).
|
||||||
|
Msg("Large file integration test memory usage")
|
||||||
|
|
||||||
|
outputFile := strings.TrimSuffix(cbzFile, ".cbz") + "_converted.cbz"
|
||||||
|
if _, err := os.Stat(outputFile); err != nil {
|
||||||
|
t.Fatalf("expected converted output file %s to exist: %v", outputFile, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// copyFile copies src to dst using streaming file I/O so that the whole
|
||||||
|
// file content is never held in memory at once, which matters for the
|
||||||
|
// large fixture used by this test.
|
||||||
|
func copyFile(src, dst string) (err error) {
|
||||||
|
in, err := os.Open(src)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open source file: %w", err)
|
||||||
|
}
|
||||||
|
defer errs.Capture(&err, in.Close, "failed to close source file")
|
||||||
|
|
||||||
|
out, err := os.Create(dst)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create destination file: %w", err)
|
||||||
|
}
|
||||||
|
defer errs.Capture(&err, out.Close, "failed to close destination file")
|
||||||
|
|
||||||
|
if _, err := io.Copy(out, in); err != nil {
|
||||||
|
return fmt.Errorf("failed to copy file contents: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
_ "image/gif"
|
_ "image/gif"
|
||||||
_ "image/jpeg"
|
_ "image/jpeg"
|
||||||
"image/png"
|
"image/png"
|
||||||
|
"os"
|
||||||
"runtime"
|
"runtime"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
@@ -28,6 +29,15 @@ type Converter struct {
|
|||||||
maxHeight int
|
maxHeight int
|
||||||
cropHeight int
|
cropHeight int
|
||||||
isPrepared bool
|
isPrepared bool
|
||||||
|
// pageWorkerGuard limits how many pages are converted (i.e. how many
|
||||||
|
// cwebp binary processes are spawned) concurrently across the whole
|
||||||
|
// application, regardless of how many chapters are being processed in
|
||||||
|
// parallel (see the --parallelism flag). Without this shared limit, the
|
||||||
|
// per-chapter worker pool (sized to runtime.NumCPU()) would be
|
||||||
|
// multiplied by the number of chapters processed concurrently, leading
|
||||||
|
// to CPU/memory oversubscription since every page conversion spawns an
|
||||||
|
// external cwebp process.
|
||||||
|
pageWorkerGuard chan struct{}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (converter *Converter) Format() (format constant.ConversionFormat) {
|
func (converter *Converter) Format() (format constant.ConversionFormat) {
|
||||||
@@ -40,6 +50,7 @@ func New() *Converter {
|
|||||||
maxHeight: 4000,
|
maxHeight: 4000,
|
||||||
cropHeight: 2000,
|
cropHeight: 2000,
|
||||||
isPrepared: false,
|
isPrepared: false,
|
||||||
|
pageWorkerGuard: make(chan struct{}, runtime.NumCPU()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -70,6 +81,18 @@ func (converter *Converter) ConvertChapter(ctx context.Context, chapter *manga.C
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Stage converted pages on disk in a temp folder instead of keeping them
|
||||||
|
// all in memory until the whole chapter is written out. This bounds
|
||||||
|
// memory usage for chapters with many/large pages; the caller is
|
||||||
|
// responsible for calling chapter.Cleanup() once the chapter has been
|
||||||
|
// written to its final CBZ file.
|
||||||
|
tempDir, err := os.MkdirTemp("", "cbzoptimizer-*")
|
||||||
|
if err != nil {
|
||||||
|
log.Error().Str("chapter", chapter.FilePath).Err(err).Msg("Failed to create staging temp folder")
|
||||||
|
return nil, fmt.Errorf("failed to create staging temp folder: %w", err)
|
||||||
|
}
|
||||||
|
chapter.TempDir = tempDir
|
||||||
|
|
||||||
var wgConvertedPages sync.WaitGroup
|
var wgConvertedPages sync.WaitGroup
|
||||||
maxGoroutines := runtime.NumCPU()
|
maxGoroutines := runtime.NumCPU()
|
||||||
|
|
||||||
@@ -80,7 +103,12 @@ func (converter *Converter) ConvertChapter(ctx context.Context, chapter *manga.C
|
|||||||
var wgPages sync.WaitGroup
|
var wgPages sync.WaitGroup
|
||||||
wgPages.Add(len(chapter.Pages))
|
wgPages.Add(len(chapter.Pages))
|
||||||
|
|
||||||
guard := make(chan struct{}, maxGoroutines)
|
// guard is shared across all chapters processed concurrently by this
|
||||||
|
// converter instance (see pageWorkerGuard doc comment) so that the total
|
||||||
|
// number of concurrently running cwebp processes stays bounded to
|
||||||
|
// runtime.NumCPU(), no matter how many chapters are being converted in
|
||||||
|
// parallel at the same time.
|
||||||
|
guard := converter.pageWorkerGuard
|
||||||
pagesMutex := sync.Mutex{}
|
pagesMutex := sync.Mutex{}
|
||||||
var pages []*manga.Page
|
var pages []*manga.Page
|
||||||
var totalPages = uint32(len(chapter.Pages))
|
var totalPages = uint32(len(chapter.Pages))
|
||||||
@@ -89,13 +117,25 @@ func (converter *Converter) ConvertChapter(ctx context.Context, chapter *manga.C
|
|||||||
Str("chapter", chapter.FilePath).
|
Str("chapter", chapter.FilePath).
|
||||||
Int("total_pages", len(chapter.Pages)).
|
Int("total_pages", len(chapter.Pages)).
|
||||||
Int("worker_count", maxGoroutines).
|
Int("worker_count", maxGoroutines).
|
||||||
|
Str("staging_dir", tempDir).
|
||||||
Msg("Initialized conversion worker pool")
|
Msg("Initialized conversion worker pool")
|
||||||
|
|
||||||
|
// failWithCleanup removes the staging temp folder before returning a
|
||||||
|
// fatal error for which no chapter is returned to the caller (and thus
|
||||||
|
// nobody else will clean it up).
|
||||||
|
failWithCleanup := func(err error) (*manga.Chapter, error) {
|
||||||
|
if removeErr := os.RemoveAll(tempDir); removeErr != nil {
|
||||||
|
log.Warn().Str("chapter", chapter.FilePath).Err(removeErr).Msg("Failed to remove staging temp folder after error")
|
||||||
|
}
|
||||||
|
chapter.TempDir = ""
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
// Check if context is already cancelled
|
// Check if context is already cancelled
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
log.Warn().Str("chapter", chapter.FilePath).Msg("Chapter conversion cancelled due to timeout")
|
log.Warn().Str("chapter", chapter.FilePath).Msg("Chapter conversion cancelled due to timeout")
|
||||||
return nil, ctx.Err()
|
return failWithCleanup(ctx.Err())
|
||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -122,7 +162,7 @@ func (converter *Converter) ConvertChapter(ctx context.Context, chapter *manga.C
|
|||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
convertedPage, err := converter.convertPage(pageToConvert, quality)
|
convertedPage, err := converter.convertPage(pageToConvert, quality, tempDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if convertedPage == nil {
|
if convertedPage == nil {
|
||||||
select {
|
select {
|
||||||
@@ -142,9 +182,14 @@ func (converter *Converter) ConvertChapter(ctx context.Context, chapter *manga.C
|
|||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
convertedPage.Page.Contents = buffer
|
if err := convertedPage.Page.Stage(tempDir, buffer, ".png"); err != nil {
|
||||||
convertedPage.Page.Extension = ".png"
|
select {
|
||||||
convertedPage.Page.Size = uint64(buffer.Len())
|
case errChan <- err:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
pagesMutex.Lock()
|
pagesMutex.Lock()
|
||||||
defer pagesMutex.Unlock()
|
defer pagesMutex.Unlock()
|
||||||
@@ -160,7 +205,7 @@ func (converter *Converter) ConvertChapter(ctx context.Context, chapter *manga.C
|
|||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
log.Warn().Str("chapter", chapter.FilePath).Msg("Chapter conversion cancelled due to timeout")
|
log.Warn().Str("chapter", chapter.FilePath).Msg("Chapter conversion cancelled due to timeout")
|
||||||
return nil, ctx.Err()
|
return failWithCleanup(ctx.Err())
|
||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -184,6 +229,7 @@ func (converter *Converter) ConvertChapter(ctx context.Context, chapter *manga.C
|
|||||||
select {
|
select {
|
||||||
case pagesChan <- manga.NewContainer(page, img, format, false):
|
case pagesChan <- manga.NewContainer(page, img, format, false):
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
|
wgConvertedPages.Done()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -195,6 +241,7 @@ func (converter *Converter) ConvertChapter(ctx context.Context, chapter *manga.C
|
|||||||
select {
|
select {
|
||||||
case pagesChan <- manga.NewContainer(page, img, format, true):
|
case pagesChan <- manga.NewContainer(page, img, format, true):
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
|
wgConvertedPages.Done()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
@@ -227,6 +274,7 @@ func (converter *Converter) ConvertChapter(ctx context.Context, chapter *manga.C
|
|||||||
select {
|
select {
|
||||||
case pagesChan <- manga.NewContainer(newPage, img, "N/A", true):
|
case pagesChan <- manga.NewContainer(newPage, img, "N/A", true):
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
|
wgConvertedPages.Done()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -248,11 +296,13 @@ func (converter *Converter) ConvertChapter(ctx context.Context, chapter *manga.C
|
|||||||
// Conversion completed successfully
|
// Conversion completed successfully
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
log.Warn().Str("chapter", chapter.FilePath).Msg("Chapter conversion cancelled due to timeout")
|
log.Warn().Str("chapter", chapter.FilePath).Msg("Chapter conversion cancelled due to timeout")
|
||||||
return nil, ctx.Err()
|
return failWithCleanup(ctx.Err())
|
||||||
}
|
}
|
||||||
|
|
||||||
close(errChan)
|
close(errChan)
|
||||||
close(guard)
|
// Note: guard is the converter's shared pageWorkerGuard and must not be
|
||||||
|
// closed here since it is reused by subsequent (and possibly concurrent)
|
||||||
|
// ConvertChapter calls.
|
||||||
|
|
||||||
var errList []error
|
var errList []error
|
||||||
for err := range errChan {
|
for err := range errChan {
|
||||||
@@ -399,7 +449,7 @@ func (converter *Converter) checkPageNeedsSplit(page *manga.Page, splitRequested
|
|||||||
return needsSplit, img, format, nil
|
return needsSplit, img, format, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (converter *Converter) convertPage(container *manga.PageContainer, quality uint8) (*manga.PageContainer, error) {
|
func (converter *Converter) convertPage(container *manga.PageContainer, quality uint8, tempDir string) (*manga.PageContainer, error) {
|
||||||
log.Debug().
|
log.Debug().
|
||||||
Uint16("page_index", container.Page.Index).
|
Uint16("page_index", container.Page.Index).
|
||||||
Str("format", container.Format).
|
Str("format", container.Format).
|
||||||
@@ -436,12 +486,22 @@ func (converter *Converter) convertPage(container *manga.PageContainer, quality
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
container.SetConverted(converted, ".webp")
|
originalSize := container.Page.Size
|
||||||
|
convertedSize := converted.Len()
|
||||||
|
|
||||||
|
if err := container.Page.Stage(tempDir, converted, ".webp"); err != nil {
|
||||||
|
log.Error().
|
||||||
|
Uint16("page_index", container.Page.Index).
|
||||||
|
Err(err).
|
||||||
|
Msg("Failed to stage converted page to disk")
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
container.HasBeenConverted = true
|
||||||
|
|
||||||
log.Debug().
|
log.Debug().
|
||||||
Uint16("page_index", container.Page.Index).
|
Uint16("page_index", container.Page.Index).
|
||||||
Int("original_size", len(container.Page.Contents.Bytes())).
|
Uint64("original_size", originalSize).
|
||||||
Int("converted_size", len(converted.Bytes())).
|
Int("converted_size", convertedSize).
|
||||||
Msg("Page conversion completed")
|
Msg("Page conversion completed")
|
||||||
|
|
||||||
return container, nil
|
return container, nil
|
||||||
|
|||||||
@@ -3,13 +3,16 @@ package webp
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"fmt"
|
||||||
"image"
|
"image"
|
||||||
"image/color"
|
"image/color"
|
||||||
"image/gif"
|
"image/gif"
|
||||||
"image/jpeg"
|
"image/jpeg"
|
||||||
"image/png"
|
"image/png"
|
||||||
|
"io"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
_ "golang.org/x/image/webp"
|
_ "golang.org/x/image/webp"
|
||||||
|
|
||||||
@@ -51,7 +54,9 @@ func encodeImage(img image.Image, format string) (*bytes.Buffer, string, error)
|
|||||||
}
|
}
|
||||||
return buf, ".gif", nil
|
return buf, ".gif", nil
|
||||||
case "webp":
|
case "webp":
|
||||||
PrepareEncoder()
|
if err := PrepareEncoder(); err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
if err := Encode(buf, img, 80); err != nil {
|
if err := Encode(buf, img, 80); err != nil {
|
||||||
return nil, "", err
|
return nil, "", err
|
||||||
}
|
}
|
||||||
@@ -82,11 +87,19 @@ func createTestPage(t *testing.T, index int, width, height int, format string) *
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validateConvertedImage(t *testing.T, page *manga.Page) {
|
func validateConvertedImage(t *testing.T, page *manga.Page) {
|
||||||
require.NotNil(t, page.Contents)
|
require.True(t, page.Contents != nil || page.TempFilePath != "", "page should have contents in memory or staged on disk")
|
||||||
require.Greater(t, page.Contents.Len(), 0)
|
require.Greater(t, page.Size, uint64(0))
|
||||||
|
|
||||||
|
reader, err := page.Open()
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer reader.Close()
|
||||||
|
|
||||||
|
data, err := io.ReadAll(reader)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Greater(t, len(data), 0)
|
||||||
|
|
||||||
// Try to decode the image
|
// Try to decode the image
|
||||||
img, format, err := image.Decode(bytes.NewReader(page.Contents.Bytes()))
|
img, format, err := image.Decode(bytes.NewReader(data))
|
||||||
require.NoError(t, err, "Failed to decode converted image")
|
require.NoError(t, err, "Failed to decode converted image")
|
||||||
|
|
||||||
if page.Extension == ".webp" {
|
if page.Extension == ".webp" {
|
||||||
@@ -287,7 +300,7 @@ func TestConverter_convertPage(t *testing.T) {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
container := manga.NewContainer(page, img, tt.format, tt.isToBeConverted)
|
container := manga.NewContainer(page, img, tt.format, tt.isToBeConverted)
|
||||||
|
|
||||||
converted, err := converter.convertPage(container, 80)
|
converted, err := converter.convertPage(container, 80, t.TempDir())
|
||||||
|
|
||||||
if tt.expectError {
|
if tt.expectError {
|
||||||
assert.Error(t, err)
|
assert.Error(t, err)
|
||||||
@@ -323,7 +336,7 @@ func TestConverter_convertPage_EncodingError(t *testing.T) {
|
|||||||
|
|
||||||
container := manga.NewContainer(corruptedPage, nil, "png", true)
|
container := manga.NewContainer(corruptedPage, nil, "png", true)
|
||||||
|
|
||||||
converted, err := converter.convertPage(container, 80)
|
converted, err := converter.convertPage(container, 80, t.TempDir())
|
||||||
|
|
||||||
// This should return nil container and error because encoding will fail with nil image
|
// This should return nil container and error because encoding will fail with nil image
|
||||||
assert.Error(t, err)
|
assert.Error(t, err)
|
||||||
@@ -422,3 +435,172 @@ func TestConverter_ConvertChapter_Timeout(t *testing.T) {
|
|||||||
assert.Nil(t, convertedChapter)
|
assert.Nil(t, convertedChapter)
|
||||||
assert.Equal(t, context.DeadlineExceeded, err)
|
assert.Equal(t, context.DeadlineExceeded, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestConverter_ConvertChapter_ManyPages_NoDeadlock tests that converting chapters with many pages
|
||||||
|
// does not cause a deadlock. This test reproduces the scenario where processing
|
||||||
|
// many files with context cancellation could cause "all goroutines are asleep - deadlock!" error.
|
||||||
|
// The fix ensures that wgConvertedPages.Done() is called when context is cancelled after Add(1).
|
||||||
|
func TestConverter_ConvertChapter_ManyPages_NoDeadlock(t *testing.T) {
|
||||||
|
converter := New()
|
||||||
|
err := converter.PrepareConverter()
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Create a chapter with many pages to increase the chance of hitting the race condition
|
||||||
|
numPages := 50
|
||||||
|
pages := make([]*manga.Page, numPages)
|
||||||
|
for i := 0; i < numPages; i++ {
|
||||||
|
pages[i] = createTestPage(t, i+1, 100, 100, "jpeg")
|
||||||
|
}
|
||||||
|
|
||||||
|
chapter := &manga.Chapter{
|
||||||
|
FilePath: "/test/chapter_many_pages.cbz",
|
||||||
|
Pages: pages,
|
||||||
|
}
|
||||||
|
|
||||||
|
progress := func(message string, current uint32, total uint32) {
|
||||||
|
// No-op progress callback
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run multiple iterations to increase the chance of hitting the race condition
|
||||||
|
for iteration := 0; iteration < 10; iteration++ {
|
||||||
|
t.Run(fmt.Sprintf("iteration_%d", iteration), func(t *testing.T) {
|
||||||
|
// Use a very short timeout to trigger context cancellation during processing
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), time.Nanosecond)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
// This should NOT deadlock - it should return quickly with context error
|
||||||
|
done := make(chan struct{})
|
||||||
|
var convertErr error
|
||||||
|
go func() {
|
||||||
|
defer close(done)
|
||||||
|
_, convertErr = converter.ConvertChapter(ctx, chapter, 80, false, progress)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Wait with a reasonable timeout - if it takes longer than 5 seconds, we have a deadlock
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
// Expected - conversion should complete (with error) quickly
|
||||||
|
assert.Error(t, convertErr, "Expected context error")
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("Deadlock detected: ConvertChapter did not return within 5 seconds")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestConverter_ConvertChapter_ManyPages_WithSplit_NoDeadlock tests that converting chapters
|
||||||
|
// with many pages and split enabled does not cause a deadlock.
|
||||||
|
func TestConverter_ConvertChapter_ManyPages_WithSplit_NoDeadlock(t *testing.T) {
|
||||||
|
converter := New()
|
||||||
|
err := converter.PrepareConverter()
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Create pages with varying heights, some requiring splits
|
||||||
|
numPages := 30
|
||||||
|
pages := make([]*manga.Page, numPages)
|
||||||
|
for i := 0; i < numPages; i++ {
|
||||||
|
height := 1000 // Normal height
|
||||||
|
if i%5 == 0 {
|
||||||
|
height = 5000 // Tall image that will be split
|
||||||
|
}
|
||||||
|
pages[i] = createTestPage(t, i+1, 100, height, "png")
|
||||||
|
}
|
||||||
|
|
||||||
|
chapter := &manga.Chapter{
|
||||||
|
FilePath: "/test/chapter_split_test.cbz",
|
||||||
|
Pages: pages,
|
||||||
|
}
|
||||||
|
|
||||||
|
progress := func(message string, current uint32, total uint32) {
|
||||||
|
// No-op progress callback
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run multiple iterations with short timeouts
|
||||||
|
for iteration := 0; iteration < 10; iteration++ {
|
||||||
|
t.Run(fmt.Sprintf("iteration_%d", iteration), func(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), time.Nanosecond)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
done := make(chan struct{})
|
||||||
|
var convertErr error
|
||||||
|
go func() {
|
||||||
|
defer close(done)
|
||||||
|
_, convertErr = converter.ConvertChapter(ctx, chapter, 80, true, progress) // split=true
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
assert.Error(t, convertErr, "Expected context error")
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("Deadlock detected: ConvertChapter with split did not return within 5 seconds")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestConverter_ConvertChapter_ConcurrentChapters_NoDeadlock simulates the scenario from the
|
||||||
|
// original bug report where multiple chapters are processed in parallel with parallelism > 1.
|
||||||
|
// This test ensures no deadlock occurs when multiple goroutines are converting chapters concurrently.
|
||||||
|
func TestConverter_ConvertChapter_ConcurrentChapters_NoDeadlock(t *testing.T) {
|
||||||
|
converter := New()
|
||||||
|
err := converter.PrepareConverter()
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Create multiple chapters, each with many pages
|
||||||
|
numChapters := 20
|
||||||
|
pagesPerChapter := 30
|
||||||
|
chapters := make([]*manga.Chapter, numChapters)
|
||||||
|
|
||||||
|
for c := 0; c < numChapters; c++ {
|
||||||
|
pages := make([]*manga.Page, pagesPerChapter)
|
||||||
|
for i := 0; i < pagesPerChapter; i++ {
|
||||||
|
pages[i] = createTestPage(t, i+1, 100, 100, "jpeg")
|
||||||
|
}
|
||||||
|
chapters[c] = &manga.Chapter{
|
||||||
|
FilePath: fmt.Sprintf("/test/chapter_%d.cbz", c+1),
|
||||||
|
Pages: pages,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
progress := func(message string, current uint32, total uint32) {}
|
||||||
|
|
||||||
|
// Process chapters concurrently with short timeouts (simulating parallelism flag)
|
||||||
|
parallelism := 4
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
semaphore := make(chan struct{}, parallelism)
|
||||||
|
|
||||||
|
// Overall test timeout
|
||||||
|
testCtx, testCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
|
defer testCancel()
|
||||||
|
|
||||||
|
for _, chapter := range chapters {
|
||||||
|
wg.Add(1)
|
||||||
|
semaphore <- struct{}{} // Acquire
|
||||||
|
|
||||||
|
go func(ch *manga.Chapter) {
|
||||||
|
defer wg.Done()
|
||||||
|
defer func() { <-semaphore }() // Release
|
||||||
|
|
||||||
|
// Use very short timeout to trigger cancellation
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), time.Nanosecond)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
// This should not deadlock
|
||||||
|
_, _ = converter.ConvertChapter(ctx, ch, 80, false, progress)
|
||||||
|
}(chapter)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for all conversions with a timeout
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
wg.Wait()
|
||||||
|
close(done)
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
// All goroutines completed successfully
|
||||||
|
case <-testCtx.Done():
|
||||||
|
t.Fatal("Deadlock detected: Concurrent chapter conversions did not complete within 30 seconds")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Vendored
+3
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:8880c8909521c3716a339a39a29a4bf1114921a8b4651ff8ca04da7ce3a54b6a
|
||||||
|
size 1001668690
|
||||||
Reference in New Issue
Block a user