refactor!: disk-first zero-copy architecture with file-to-file cwebp conversion

BREAKING CHANGE: Complete refactoring of the conversion pipeline.

- Replace in-memory Page/PageContainer with disk-based PageFile struct
- Extract archives to temp directory instead of loading into memory
- Use cwebp InputFile/OutputFile for zero-copy file-to-file conversion
- Use cwebp -crop for splitting (no Go-side image decode needed)
- Check if already converted before extracting (fast pre-check)
- Remove oliamb/cutter dependency (replaced by cwebp -crop)
- Remove page_container.go (no longer needed)
- Add comprehensive tests with improved coverage

Co-authored-by: Belphemur <197810+Belphemur@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-07-04 22:18:22 +00:00
committed by GitHub
co-authored by Belphemur
parent e87e73eb28
commit 164dc577b9
21 changed files with 1735 additions and 1386 deletions
+14 -8
View File
@@ -11,13 +11,19 @@ import (
"github.com/samber/lo"
)
// Converter defines the interface for image format converters.
// All operations are file-to-file: no image data is held in memory in the happy path.
type Converter interface {
// Format of the converter
Format() (format constant.ConversionFormat)
// ConvertChapter converts a manga chapter to the specified format.
//
// Returns partial success where some pages are converted and some are not.
// Format returns the output format of this converter.
Format() constant.ConversionFormat
// ConvertChapter converts all pages in a chapter from their source files to
// the target format. Pages are processed in parallel (bounded by CPU count).
// On success, chapter.Pages is updated with converted PageFile entries.
// Returns partial success (non-fatal errors) via errors.PageIgnoredError.
ConvertChapter(ctx context.Context, chapter *manga.Chapter, quality uint8, split bool, progress func(message string, current uint32, total uint32)) (*manga.Chapter, error)
// PrepareConverter ensures the external encoder binary is available.
PrepareConverter() error
}
@@ -30,8 +36,8 @@ func Available() []constant.ConversionFormat {
return lo.Keys(converters)
}
// Get returns a packer by name.
// If the packer is not available, an error is returned.
// Get returns a converter by format name.
// If the converter is not available, an error is returned.
var Get = getConverter
func getConverter(name constant.ConversionFormat) (Converter, error) {
@@ -39,7 +45,7 @@ func getConverter(name constant.ConversionFormat) (Converter, error) {
return converter, nil
}
return nil, fmt.Errorf("unkown converter \"%s\", available options are %s", name, strings.Join(lo.Map(Available(), func(item constant.ConversionFormat, index int) string {
return nil, fmt.Errorf("unknown converter \"%s\", available options are %s", name, strings.Join(lo.Map(Available(), func(item constant.ConversionFormat, index int) string {
return item.String()
}), ", "))
}
+136 -173
View File
@@ -1,24 +1,23 @@
package converter
import (
"bytes"
"context"
"fmt"
"image"
"image/jpeg"
"os"
"path/filepath"
"testing"
"github.com/belphemur/CBZOptimizer/v2/internal/manga"
"github.com/belphemur/CBZOptimizer/v2/internal/utils/errs"
)
func TestConvertChapter(t *testing.T) {
testCases := []struct {
name string
genTestChapter func(path string, isSplit bool) (*manga.Chapter, []string, error)
split bool
expectError bool
name string
genTestChapter func(t *testing.T, dir string) (*manga.Chapter, []string)
split bool
expectError bool
}{
{
name: "All split pages",
@@ -27,7 +26,7 @@ func TestConvertChapter(t *testing.T) {
},
{
name: "Big Pages, no split",
genTestChapter: genHugePage,
genTestChapter: genHugePageNoSplit,
split: false,
expectError: true,
},
@@ -47,53 +46,41 @@ func TestConvertChapter(t *testing.T) {
split: false,
expectError: true,
},
{
name: "Two corrupted pages",
genTestChapter: genTwoCorrupted,
split: false,
expectError: true,
},
}
// Load test genTestChapter from testdata
temp, err := os.CreateTemp("", "test_chapter_*.cbz")
if err != nil {
t.Fatalf("failed to create temporary file: %v", err)
}
defer errs.CaptureGeneric(&err, os.Remove, temp.Name(), "failed to remove temporary file")
for _, converter := range Available() {
converter, err := Get(converter)
for _, converterFormat := range Available() {
conv, err := Get(converterFormat)
if err != nil {
t.Fatalf("failed to get converter: %v", err)
}
t.Run(converter.Format().String(), func(t *testing.T) {
t.Run(conv.Format().String(), func(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
chapter, expectedExtensions, err := tc.genTestChapter(temp.Name(), tc.split)
if err != nil {
t.Fatalf("failed to load test genTestChapter: %v", err)
}
tempDir := t.TempDir()
chapter, expectedExtensions := tc.genTestChapter(t, tempDir)
quality := uint8(80)
progress := func(msg string, current uint32, total uint32) {
t.Log(msg)
}
convertedChapter, err := converter.ConvertChapter(context.Background(), chapter, quality, tc.split, progress)
convertedChapter, err := conv.ConvertChapter(context.Background(), chapter, quality, tc.split, progress)
if err != nil && !tc.expectError {
t.Fatalf("failed to convert genTestChapter: %v", err)
t.Fatalf("failed to convert chapter: %v", err)
}
if convertedChapter == nil {
t.Fatal("convertedChapter is nil")
}
if len(convertedChapter.Pages) == 0 {
t.Fatalf("no pages were converted")
t.Fatal("no pages were converted")
}
if len(convertedChapter.Pages) != len(expectedExtensions) {
t.Fatalf("converted chapter has %d pages but expected %d", len(convertedChapter.Pages), len(expectedExtensions))
}
// Check each page's extension against the expected array
for i, page := range convertedChapter.Pages {
expectedExt := expectedExtensions[i]
if page.Extension != expectedExt {
@@ -106,176 +93,152 @@ func TestConvertChapter(t *testing.T) {
}
}
func genHugePage(path string, isSplit bool) (*manga.Chapter, []string, error) {
file, err := os.Open(path)
// createTestImageFile creates a JPEG image file at the given path with the specified dimensions.
func createTestImageFile(t *testing.T, path string, width, height int) {
t.Helper()
img := image.NewRGBA(image.Rect(0, 0, width, height))
f, err := os.Create(path)
if err != nil {
return nil, nil, err
t.Fatal(err)
}
defer errs.Capture(&err, file.Close, "failed to close file")
var pages []*manga.Page
expectedExtensions := []string{".jpg"} // One image that's generated as JPEG
if isSplit {
expectedExtensions = []string{".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".webp"}
}
// Create one tall page
img := image.NewRGBA(image.Rect(0, 0, 1, 17000))
buf := new(bytes.Buffer)
err = jpeg.Encode(buf, img, nil)
err = jpeg.Encode(f, img, nil)
_ = f.Close()
if err != nil {
return nil, nil, err
t.Fatal(err)
}
page := &manga.Page{
Index: 0,
Contents: buf,
Extension: ".jpg",
}
pages = append(pages, page)
return &manga.Chapter{
FilePath: path,
Pages: pages,
}, expectedExtensions, nil
}
func genSmallPages(path string, isSplit bool) (*manga.Chapter, []string, error) {
file, err := os.Open(path)
if err != nil {
return nil, nil, err
func genHugePage(t *testing.T, dir string) (*manga.Chapter, []string) {
inputDir := filepath.Join(dir, "input")
if err := os.MkdirAll(inputDir, 0755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(filepath.Join(dir, "output"), 0755); err != nil {
t.Fatal(err)
}
defer errs.Capture(&err, file.Close, "failed to close file")
var pages []*manga.Page
for i := 0; i < 5; i++ { // Assuming there are 5 pages for the test
img := image.NewRGBA(image.Rect(0, 0, 300, 1000))
buf := new(bytes.Buffer)
err = jpeg.Encode(buf, img, nil)
if err != nil {
return nil, nil, err
}
page := &manga.Page{
pagePath := filepath.Join(inputDir, "0000.jpg")
createTestImageFile(t, pagePath, 1, 17000)
chapter := &manga.Chapter{
FilePath: filepath.Join(dir, "test.cbz"),
TempDir: dir,
Pages: []*manga.PageFile{
{Index: 0, Extension: ".jpg", FilePath: pagePath},
},
}
// With split: 17000/2000 = 9 parts (8*2000 + 1*1000)
// Without split: page > webpMaxHeight → kept as .jpg with error
// The caller decides split=true or split=false and we return expectations
// for the split=true case (this test case is always called with split=true)
expectedExtensions := []string{".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".webp"}
return chapter, expectedExtensions
}
func genHugePageNoSplit(t *testing.T, dir string) (*manga.Chapter, []string) {
inputDir := filepath.Join(dir, "input")
if err := os.MkdirAll(inputDir, 0755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(filepath.Join(dir, "output"), 0755); err != nil {
t.Fatal(err)
}
pagePath := filepath.Join(inputDir, "0000.jpg")
createTestImageFile(t, pagePath, 1, 17000)
chapter := &manga.Chapter{
FilePath: filepath.Join(dir, "test.cbz"),
TempDir: dir,
Pages: []*manga.PageFile{
{Index: 0, Extension: ".jpg", FilePath: pagePath},
},
}
// Without split: page > webpMaxHeight → kept as .jpg with error
return chapter, []string{".jpg"}
}
func genSmallPages(t *testing.T, dir string) (*manga.Chapter, []string) {
inputDir := filepath.Join(dir, "input")
_ = os.MkdirAll(inputDir, 0755)
_ = os.MkdirAll(filepath.Join(dir, "output"), 0755)
var pages []*manga.PageFile
for i := 0; i < 5; i++ {
pagePath := filepath.Join(inputDir, fmt.Sprintf("%04d.jpg", i))
createTestImageFile(t, pagePath, 300, 1000)
pages = append(pages, &manga.PageFile{
Index: uint16(i),
Contents: buf,
Extension: ".jpg",
}
pages = append(pages, page)
FilePath: pagePath,
})
}
return &manga.Chapter{
FilePath: path,
chapter := &manga.Chapter{
FilePath: filepath.Join(dir, "test.cbz"),
TempDir: dir,
Pages: pages,
}, []string{".webp", ".webp", ".webp", ".webp", ".webp"}, nil
}
return chapter, []string{".webp", ".webp", ".webp", ".webp", ".webp"}
}
func genMixSmallBig(path string, isSplit bool) (*manga.Chapter, []string, error) {
file, err := os.Open(path)
if err != nil {
return nil, nil, err
}
defer errs.Capture(&err, file.Close, "failed to close file")
func genMixSmallBig(t *testing.T, dir string) (*manga.Chapter, []string) {
inputDir := filepath.Join(dir, "input")
_ = os.MkdirAll(inputDir, 0755)
_ = os.MkdirAll(filepath.Join(dir, "output"), 0755)
var pages []*manga.Page
for i := 0; i < 5; i++ { // Assuming there are 5 pages for the test
img := image.NewRGBA(image.Rect(0, 0, 300, 1000*(i+1)))
buf := new(bytes.Buffer)
err := jpeg.Encode(buf, img, nil)
if err != nil {
return nil, nil, err
}
page := &manga.Page{
var pages []*manga.PageFile
for i := 0; i < 5; i++ {
pagePath := filepath.Join(inputDir, fmt.Sprintf("%04d.jpg", i))
createTestImageFile(t, pagePath, 300, 1000*(i+1))
pages = append(pages, &manga.PageFile{
Index: uint16(i),
Contents: buf,
Extension: ".jpg",
}
pages = append(pages, page)
}
expectedExtensions := []string{".webp", ".webp", ".webp", ".webp", ".webp"}
if isSplit {
expectedExtensions = []string{".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".webp"}
FilePath: pagePath,
})
}
return &manga.Chapter{
FilePath: path,
chapter := &manga.Chapter{
FilePath: filepath.Join(dir, "test.cbz"),
TempDir: dir,
Pages: pages,
}, expectedExtensions, nil
}
// Pages heights: 1000, 2000, 3000, 4000, 5000
// With new disk-first architecture: cwebp handles all these directly
// (all < 16383 webp max), no splitting needed
return chapter, []string{".webp", ".webp", ".webp", ".webp", ".webp"}
}
func genMixSmallHuge(path string, isSplit bool) (*manga.Chapter, []string, error) {
file, err := os.Open(path)
if err != nil {
return nil, nil, err
}
defer errs.Capture(&err, file.Close, "failed to close file")
func genMixSmallHuge(t *testing.T, dir string) (*manga.Chapter, []string) {
inputDir := filepath.Join(dir, "input")
_ = os.MkdirAll(inputDir, 0755)
_ = os.MkdirAll(filepath.Join(dir, "output"), 0755)
var pages []*manga.Page
for i := 0; i < 10; i++ { // Assuming there are 5 pages for the test
img := image.NewRGBA(image.Rect(0, 0, 1, 2000*(i+1)))
buf := new(bytes.Buffer)
err := jpeg.Encode(buf, img, nil)
if err != nil {
return nil, nil, err
}
page := &manga.Page{
var pages []*manga.PageFile
for i := 0; i < 10; i++ {
pagePath := filepath.Join(inputDir, fmt.Sprintf("%04d.jpg", i))
createTestImageFile(t, pagePath, 1, 2000*(i+1))
pages = append(pages, &manga.PageFile{
Index: uint16(i),
Contents: buf,
Extension: ".jpg",
}
pages = append(pages, page)
FilePath: pagePath,
})
}
return &manga.Chapter{
FilePath: path,
chapter := &manga.Chapter{
FilePath: filepath.Join(dir, "test.cbz"),
TempDir: dir,
Pages: pages,
}, []string{".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".jpg", ".jpg"}, nil
}
func genTwoCorrupted(path string, isSplit bool) (*manga.Chapter, []string, error) {
file, err := os.Open(path)
if err != nil {
return nil, nil, err
}
defer errs.Capture(&err, file.Close, "failed to close file")
var pages []*manga.Page
numPages := 5
corruptedIndices := []int{2, 4} // Pages 2 and 4 are too tall to convert without splitting
for i := 0; i < numPages; i++ {
var buf *bytes.Buffer
var ext string
isCorrupted := false
for _, ci := range corruptedIndices {
if i == ci {
isCorrupted = true
break
}
}
if isCorrupted {
buf = bytes.NewBufferString("corrupted data") // Invalid data, can't decode as image
ext = ".jpg"
} else {
img := image.NewRGBA(image.Rect(0, 0, 300, 1000))
buf = new(bytes.Buffer)
err = jpeg.Encode(buf, img, nil)
if err != nil {
return nil, nil, err
}
ext = ".jpg"
}
page := &manga.Page{
Index: uint16(i),
Contents: buf,
Extension: ext,
}
pages = append(pages, page)
}
// Expected: small pages to .webp, corrupted pages to .jpg (kept as is)
expectedExtensions := []string{".webp", ".webp", ".jpg", ".webp", ".jpg"}
// Even with split, corrupted pages can't be decoded so stay as is
return &manga.Chapter{
FilePath: path,
Pages: pages,
}, expectedExtensions, nil
// Heights: 2000, 4000, 6000, 8000, 10000, 12000, 14000, 16000, 18000, 20000
// Without split, pages > webpMaxHeight (16383) are kept as .jpg
// Pages 0-7 (2000-16000): should convert to .webp
// Pages 8-9 (18000, 20000): > 16383, no split → kept as .jpg
return chapter, []string{".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".jpg", ".jpg"}
}
+216 -390
View File
@@ -1,25 +1,25 @@
package webp
import (
"bytes"
"context"
"errors"
"fmt"
"image"
_ "image/gif"
_ "image/jpeg"
"image/png"
_ "image/png"
"os"
"path/filepath"
"runtime"
"strings"
"sort"
"sync"
"sync/atomic"
"github.com/belphemur/CBZOptimizer/v2/internal/manga"
"github.com/belphemur/CBZOptimizer/v2/pkg/converter/constant"
converterrors "github.com/belphemur/CBZOptimizer/v2/pkg/converter/errors"
"github.com/oliamb/cutter"
"github.com/rs/zerolog/log"
"golang.org/x/exp/slices"
_ "golang.org/x/image/webp"
)
@@ -29,24 +29,16 @@ type Converter struct {
maxHeight int
cropHeight int
isPrepared bool
// pageWorkerGuard limits how many pages are converted (i.e. how many
// cwebp binary processes are spawned) concurrently across the whole
// application, regardless of how many chapters are being processed in
// parallel (see the --parallelism flag). Without this shared limit, the
// per-chapter worker pool (sized to runtime.NumCPU()) would be
// multiplied by the number of chapters processed concurrently, leading
// to CPU/memory oversubscription since every page conversion spawns an
// external cwebp process.
// pageWorkerGuard limits concurrent cwebp processes across all chapters.
pageWorkerGuard chan struct{}
}
func (converter *Converter) Format() (format constant.ConversionFormat) {
func (converter *Converter) Format() constant.ConversionFormat {
return constant.WebP
}
func New() *Converter {
return &Converter{
//maxHeight: 16383 / 2,
maxHeight: 4000,
cropHeight: 2000,
isPrepared: false,
@@ -66,456 +58,290 @@ func (converter *Converter) PrepareConverter() error {
return nil
}
// ConvertChapter converts all pages in a chapter using file-to-file cwebp operations.
// In the happy path, no image data is loaded into Go memory.
// Splitting is attempted only if direct conversion fails due to dimension limits.
func (converter *Converter) ConvertChapter(ctx context.Context, chapter *manga.Chapter, quality uint8, split bool, progress func(message string, current uint32, total uint32)) (*manga.Chapter, error) {
log.Debug().
Str("chapter", chapter.FilePath).
Int("pages", len(chapter.Pages)).
Uint8("quality", quality).
Bool("split", split).
Int("max_goroutines", runtime.NumCPU()).
Msg("Starting chapter conversion")
Msg("Starting file-to-file chapter conversion")
err := converter.PrepareConverter()
if err != nil {
log.Error().Str("chapter", chapter.FilePath).Err(err).Msg("Failed to prepare converter")
return nil, err
}
// Stage converted pages on disk in a temp folder instead of keeping them
// all in memory until the whole chapter is written out. This bounds
// memory usage for chapters with many/large pages; the caller is
// responsible for calling chapter.Cleanup() once the chapter has been
// written to its final CBZ file.
tempDir, err := os.MkdirTemp("", "cbzoptimizer-*")
if err != nil {
log.Error().Str("chapter", chapter.FilePath).Err(err).Msg("Failed to create staging temp folder")
return nil, fmt.Errorf("failed to create staging temp folder: %w", err)
}
chapter.TempDir = tempDir
var wgConvertedPages sync.WaitGroup
maxGoroutines := runtime.NumCPU()
pagesChan := make(chan *manga.PageContainer, maxGoroutines)
errChan := make(chan error, maxGoroutines)
doneChan := make(chan struct{})
var wgPages sync.WaitGroup
wgPages.Add(len(chapter.Pages))
// guard is shared across all chapters processed concurrently by this
// converter instance (see pageWorkerGuard doc comment) so that the total
// number of concurrently running cwebp processes stays bounded to
// runtime.NumCPU(), no matter how many chapters are being converted in
// parallel at the same time.
guard := converter.pageWorkerGuard
pagesMutex := sync.Mutex{}
var pages []*manga.Page
var totalPages = uint32(len(chapter.Pages))
log.Debug().
Str("chapter", chapter.FilePath).
Int("total_pages", len(chapter.Pages)).
Int("worker_count", maxGoroutines).
Str("staging_dir", tempDir).
Msg("Initialized conversion worker pool")
// failWithCleanup removes the staging temp folder before returning a
// fatal error for which no chapter is returned to the caller (and thus
// nobody else will clean it up).
failWithCleanup := func(err error) (*manga.Chapter, error) {
if removeErr := os.RemoveAll(tempDir); removeErr != nil {
log.Warn().Str("chapter", chapter.FilePath).Err(removeErr).Msg("Failed to remove staging temp folder after error")
}
chapter.TempDir = ""
return nil, err
// Create output directory for converted files
outputDir := filepath.Join(chapter.TempDir, "output")
if err := os.MkdirAll(outputDir, 0755); err != nil {
return nil, fmt.Errorf("failed to create output directory: %w", err)
}
// Check if context is already cancelled
// Check for early context cancellation
select {
case <-ctx.Done():
log.Warn().Str("chapter", chapter.FilePath).Msg("Chapter conversion cancelled due to timeout")
return failWithCleanup(ctx.Err())
return nil, ctx.Err()
default:
}
// Start the worker pool
go func() {
defer close(doneChan)
for page := range pagesChan {
select {
case <-ctx.Done():
return
case guard <- struct{}{}: // would block if guard channel is already filled
}
guard := converter.pageWorkerGuard
var totalPages atomic.Uint32
totalPages.Store(uint32(len(chapter.Pages)))
go func(pageToConvert *manga.PageContainer) {
defer func() {
wgConvertedPages.Done()
<-guard
}()
// Check context cancellation before processing
select {
case <-ctx.Done():
return
default:
}
convertedPage, err := converter.convertPage(pageToConvert, quality, tempDir)
if err != nil {
if convertedPage == nil {
select {
case errChan <- err:
case <-ctx.Done():
return
}
return
}
buffer := new(bytes.Buffer)
err := png.Encode(buffer, convertedPage.Image)
if err != nil {
select {
case errChan <- err:
case <-ctx.Done():
return
}
return
}
if err := convertedPage.Page.Stage(tempDir, buffer, ".png"); err != nil {
select {
case errChan <- err:
case <-ctx.Done():
return
}
return
}
}
pagesMutex.Lock()
defer pagesMutex.Unlock()
pages = append(pages, convertedPage.Page)
currentTotalPages := atomic.LoadUint32(&totalPages)
progress(fmt.Sprintf("Converted %d/%d pages to %s format", len(pages), currentTotalPages, converter.Format()), uint32(len(pages)), currentTotalPages)
}(page)
}
}()
// Process pages
for _, page := range chapter.Pages {
select {
case <-ctx.Done():
log.Warn().Str("chapter", chapter.FilePath).Msg("Chapter conversion cancelled due to timeout")
return failWithCleanup(ctx.Err())
default:
}
go func(page *manga.Page) {
defer wgPages.Done()
splitNeeded, img, format, err := converter.checkPageNeedsSplit(page, split)
if err != nil {
var pageIgnoredError *converterrors.PageIgnoredError
if errors.As(err, &pageIgnoredError) {
log.Info().Err(err).Msg("Page ignored due to image decode error")
}
select {
case errChan <- err:
case <-ctx.Done():
return
}
wgConvertedPages.Add(1)
select {
case pagesChan <- manga.NewContainer(page, img, format, false):
case <-ctx.Done():
wgConvertedPages.Done()
return
}
return
}
if !splitNeeded {
wgConvertedPages.Add(1)
select {
case pagesChan <- manga.NewContainer(page, img, format, true):
case <-ctx.Done():
wgConvertedPages.Done()
return
}
return
}
images, err := converter.cropImage(img)
if err != nil {
select {
case errChan <- err:
case <-ctx.Done():
return
}
return
}
atomic.AddUint32(&totalPages, uint32(len(images)-1))
for i, img := range images {
select {
case <-ctx.Done():
return
default:
}
newPage := &manga.Page{
Index: page.Index,
IsSplitted: true,
SplitPartIndex: uint16(i),
}
wgConvertedPages.Add(1)
select {
case pagesChan <- manga.NewContainer(newPage, img, "N/A", true):
case <-ctx.Done():
wgConvertedPages.Done()
return
}
}
}(page)
type pageResult struct {
pages []*manga.PageFile
err error
}
wgPages.Wait()
close(pagesChan)
results := make([]pageResult, len(chapter.Pages))
var wg sync.WaitGroup
var convertedCount atomic.Uint32
// Wait for all conversions to complete or context cancellation
for i, page := range chapter.Pages {
wg.Add(1)
go func(idx int, p *manga.PageFile) {
defer wg.Done()
// Check context before acquiring worker slot
select {
case <-ctx.Done():
results[idx] = pageResult{err: ctx.Err()}
return
case guard <- struct{}{}:
}
defer func() { <-guard }()
// Check context after acquiring worker slot
select {
case <-ctx.Done():
results[idx] = pageResult{err: ctx.Err()}
return
default:
}
pages, err := converter.convertPageFile(ctx, p, outputDir, quality, split)
results[idx] = pageResult{pages: pages, err: err}
current := convertedCount.Add(1)
total := totalPages.Load()
progress(fmt.Sprintf("Converted %d/%d pages to %s format", current, total, converter.Format()), current, total)
}(i, page)
}
// Wait for completion or context cancellation
done := make(chan struct{})
go func() {
defer close(done)
wgConvertedPages.Wait()
wg.Wait()
close(done)
}()
select {
case <-done:
// Conversion completed successfully
case <-ctx.Done():
log.Warn().Str("chapter", chapter.FilePath).Msg("Chapter conversion cancelled due to timeout")
return failWithCleanup(ctx.Err())
// Wait for in-flight goroutines to finish
<-done
return nil, ctx.Err()
}
close(errChan)
// Note: guard is the converter's shared pageWorkerGuard and must not be
// closed here since it is reused by subsequent (and possibly concurrent)
// ConvertChapter calls.
// Collect results
var convertedPages []*manga.PageFile
var errList []error
for err := range errChan {
errList = append(errList, err)
for _, result := range results {
if result.err != nil {
if errors.Is(result.err, context.DeadlineExceeded) || errors.Is(result.err, context.Canceled) {
return nil, result.err
}
errList = append(errList, result.err)
}
if result.pages != nil {
convertedPages = append(convertedPages, result.pages...)
}
}
var aggregatedError error = nil
if len(convertedPages) == 0 {
if len(errList) > 0 {
return nil, errors.Join(errList...)
}
return nil, fmt.Errorf("no pages were converted")
}
// Sort pages by index and split part
sort.Slice(convertedPages, func(i, j int) bool {
a, b := convertedPages[i], convertedPages[j]
if a.Index == b.Index {
return a.SplitPartIndex < b.SplitPartIndex
}
return a.Index < b.Index
})
chapter.Pages = convertedPages
var aggregatedError error
if len(errList) > 0 {
aggregatedError = errors.Join(errList...)
log.Debug().
Str("chapter", chapter.FilePath).
Int("error_count", len(errList)).
Err(errors.Join(errList...)).
Msg("Conversion completed with errors")
} else {
log.Debug().
Str("chapter", chapter.FilePath).
Int("pages_converted", len(pages)).
Msg("Conversion completed successfully")
}
slices.SortFunc(pages, func(a, b *manga.Page) int {
if a.Index == b.Index {
return int(a.SplitPartIndex) - int(b.SplitPartIndex)
}
return int(a.Index) - int(b.Index)
})
chapter.Pages = pages
log.Debug().
Str("chapter", chapter.FilePath).
Int("final_page_count", len(pages)).
Msg("Pages sorted and chapter updated")
runtime.GC()
log.Debug().Str("chapter", chapter.FilePath).Msg("Garbage collection completed")
Int("converted_pages", len(convertedPages)).
Msg("Chapter conversion completed")
return chapter, aggregatedError
}
func (converter *Converter) cropImage(img image.Image) ([]image.Image, error) {
bounds := img.Bounds()
height := bounds.Dy()
width := bounds.Dx()
// convertPageFile converts a single page file to WebP format.
// Returns the converted page(s) — multiple if splitting was needed.
func (converter *Converter) convertPageFile(ctx context.Context, page *manga.PageFile, outputDir string, quality uint8, split bool) ([]*manga.PageFile, error) {
log.Debug().
Uint16("page_index", page.Index).
Str("input", page.FilePath).
Msg("Converting page file")
// If the page is already WebP, just return it as-is
if strings.ToLower(page.Extension) == ".webp" {
log.Debug().Uint16("page_index", page.Index).Msg("Page already WebP, skipping")
return []*manga.PageFile{page}, nil
}
// Try direct file-to-file conversion first (happy path — no memory allocation)
outputPath := filepath.Join(outputDir, fmt.Sprintf("%04d.webp", page.Index))
err := EncodeFile(page.FilePath, outputPath, uint(quality))
if err == nil {
// Success! No image decoding needed.
return []*manga.PageFile{{
Index: page.Index,
Extension: ".webp",
FilePath: outputPath,
}}, nil
}
// Direct conversion failed. Check if it's a dimension issue.
log.Debug().
Uint16("page_index", page.Index).
Err(err).
Msg("Direct conversion failed, checking dimensions")
// Read just the image header to get dimensions (no full decode)
width, height, decodeErr := getImageDimensions(page.FilePath)
if decodeErr != nil {
// Can't even read the image header — keep the original file
log.Info().
Uint16("page_index", page.Index).
Err(decodeErr).
Msg("Cannot decode image, keeping original")
return []*manga.PageFile{page}, converterrors.NewPageIgnored(
fmt.Sprintf("page %d: failed to decode image (%s)", page.Index, decodeErr.Error()))
}
log.Debug().
Uint16("page_index", page.Index).
Int("width", width).
Int("height", height).
Msg("Image dimensions read")
// If height exceeds WebP max and split is not enabled, keep original
if height >= webpMaxHeight && !split {
log.Info().
Uint16("page_index", page.Index).
Int("height", height).
Msg("Page too tall for WebP, keeping original")
return []*manga.PageFile{page}, converterrors.NewPageIgnored(
fmt.Sprintf("page %d is too tall [max: %dpx] to be converted to webp format", page.Index, webpMaxHeight))
}
// If height exceeds our split threshold and split is enabled, use cwebp -crop
if height >= converter.maxHeight && split {
return converter.splitAndConvert(ctx, page, outputDir, quality, width, height)
}
// Height is within limits but conversion still failed for another reason.
// Keep the original file.
log.Warn().
Uint16("page_index", page.Index).
Err(err).
Msg("Conversion failed for non-dimension reason, keeping original")
return []*manga.PageFile{page}, converterrors.NewPageIgnored(
fmt.Sprintf("page %d: conversion failed (%s)", page.Index, err.Error()))
}
// splitAndConvert splits a tall image into multiple parts using cwebp -crop
// and converts each part. No Go-side image decode is needed.
func (converter *Converter) splitAndConvert(ctx context.Context, page *manga.PageFile, outputDir string, quality uint8, width, height int) ([]*manga.PageFile, error) {
log.Debug().
Uint16("page_index", page.Index).
Int("width", width).
Int("height", height).
Int("crop_height", converter.cropHeight).
Msg("Splitting and converting page using cwebp -crop")
numParts := height / converter.cropHeight
if height%converter.cropHeight != 0 {
numParts++
}
log.Debug().
Int("original_width", width).
Int("original_height", height).
Int("crop_height", converter.cropHeight).
Int("num_parts", numParts).
Msg("Starting image cropping for page splitting")
parts := make([]image.Image, numParts)
var pages []*manga.PageFile
for i := 0; i < numParts; i++ {
select {
case <-ctx.Done():
return pages, ctx.Err()
default:
}
yOffset := i * converter.cropHeight
partHeight := converter.cropHeight
if i == numParts-1 {
partHeight = height - i*converter.cropHeight
partHeight = height - yOffset
}
log.Debug().
Int("part_index", i).
Int("part_height", partHeight).
Int("y_offset", i*converter.cropHeight).
Msg("Cropping image part")
outputPath := filepath.Join(outputDir, fmt.Sprintf("%04d-%02d.webp", page.Index, i))
err := EncodeFileWithCrop(page.FilePath, outputPath, uint(quality), 0, yOffset, width, partHeight)
part, err := cutter.Crop(img, cutter.Config{
Width: bounds.Dx(),
Height: partHeight,
Anchor: image.Point{Y: i * converter.cropHeight},
Mode: cutter.TopLeft,
})
if err != nil {
log.Error().
Int("part_index", i).
Uint16("page_index", page.Index).
Int("part", i).
Err(err).
Msg("Failed to crop image part")
return nil, fmt.Errorf("error cropping part %d: %v", i+1, err)
Msg("Failed to convert split part")
return nil, fmt.Errorf("failed to convert split part %d of page %d: %w", i, page.Index, err)
}
parts[i] = part
log.Debug().
Int("part_index", i).
Int("cropped_width", part.Bounds().Dx()).
Int("cropped_height", part.Bounds().Dy()).
Msg("Image part cropped successfully")
pages = append(pages, &manga.PageFile{
Index: page.Index,
Extension: ".webp",
FilePath: outputPath,
IsSplitted: true,
SplitPartIndex: uint16(i),
})
}
log.Debug().
Int("total_parts", len(parts)).
Msg("Image cropping completed")
return parts, nil
}
func (converter *Converter) checkPageNeedsSplit(page *manga.Page, splitRequested bool) (bool, image.Image, string, error) {
log.Debug().
Uint16("page_index", page.Index).
Bool("split_requested", splitRequested).
Int("page_size", len(page.Contents.Bytes())).
Msg("Analyzing page for splitting")
reader := bytes.NewBuffer(page.Contents.Bytes())
img, format, err := image.Decode(reader)
if err != nil {
log.Debug().Uint16("page_index", page.Index).Err(err).Msg("Failed to decode page image")
return false, nil, format, converterrors.NewPageIgnored(fmt.Sprintf("page %d: failed to decode image (%s)", page.Index, err.Error()))
}
bounds := img.Bounds()
height := bounds.Dy()
width := bounds.Dx()
log.Debug().
Uint16("page_index", page.Index).
Int("width", width).
Int("height", height).
Str("format", format).
Int("max_height", converter.maxHeight).
Int("webp_max_height", webpMaxHeight).
Msg("Page dimensions analyzed")
Int("parts", len(pages)).
Msg("Split conversion completed")
if height >= webpMaxHeight && !splitRequested {
log.Debug().
Uint16("page_index", page.Index).
Int("height", height).
Int("webp_max", webpMaxHeight).
Msg("Page too tall for WebP format, would be ignored")
return false, img, format, converterrors.NewPageIgnored(fmt.Sprintf("page %d is too tall [max: %dpx] to be converted to webp format", page.Index, webpMaxHeight))
}
needsSplit := height >= converter.maxHeight && splitRequested
log.Debug().
Uint16("page_index", page.Index).
Bool("needs_split", needsSplit).
Msg("Page splitting decision made")
return needsSplit, img, format, nil
return pages, nil
}
func (converter *Converter) convertPage(container *manga.PageContainer, quality uint8, tempDir string) (*manga.PageContainer, error) {
log.Debug().
Uint16("page_index", container.Page.Index).
Str("format", container.Format).
Bool("to_be_converted", container.IsToBeConverted).
Uint8("quality", quality).
Msg("Converting page")
// Fix WebP format detection (case insensitive)
if container.Format == "webp" || container.Format == "WEBP" {
log.Debug().
Uint16("page_index", container.Page.Index).
Msg("Page already in WebP format, skipping conversion")
container.Page.Extension = ".webp"
return container, nil
}
if !container.IsToBeConverted {
log.Debug().
Uint16("page_index", container.Page.Index).
Msg("Page marked as not to be converted, skipping")
return container, nil
}
log.Debug().
Uint16("page_index", container.Page.Index).
Uint8("quality", quality).
Msg("Encoding page to WebP format")
converted, err := converter.convert(container.Image, uint(quality))
// getImageDimensions reads only the image header to determine dimensions.
// This is much cheaper than a full image.Decode — only a few bytes are read.
func getImageDimensions(filePath string) (width, height int, err error) {
f, err := os.Open(filePath)
if err != nil {
log.Error().
Uint16("page_index", container.Page.Index).
Err(err).
Msg("Failed to convert page to WebP")
return nil, err
return 0, 0, err
}
defer func() { _ = f.Close() }()
originalSize := container.Page.Size
convertedSize := converted.Len()
if err := container.Page.Stage(tempDir, converted, ".webp"); err != nil {
log.Error().
Uint16("page_index", container.Page.Index).
Err(err).
Msg("Failed to stage converted page to disk")
return nil, err
}
container.HasBeenConverted = true
log.Debug().
Uint16("page_index", container.Page.Index).
Uint64("original_size", originalSize).
Int("converted_size", convertedSize).
Msg("Page conversion completed")
return container, nil
}
// convert converts an image to the WebP format. It decodes the image from the input buffer,
// encodes it as a WebP file using the webp.Encode() function, and returns the resulting WebP
// file as a bytes.Buffer.
func (converter *Converter) convert(image image.Image, quality uint) (*bytes.Buffer, error) {
var buf bytes.Buffer
err := Encode(&buf, image, quality)
config, _, err := image.DecodeConfig(f)
if err != nil {
return nil, err
return 0, 0, err
}
return &buf, nil
return config.Width, config.Height, nil
}
+289 -408
View File
@@ -1,138 +1,61 @@
package webp
import (
"bytes"
"context"
"fmt"
"image"
"image/color"
"image/gif"
"image/jpeg"
"image/png"
"io"
"os"
"path/filepath"
"sync"
"testing"
"time"
_ "golang.org/x/image/webp"
"github.com/belphemur/CBZOptimizer/v2/internal/manga"
"github.com/belphemur/CBZOptimizer/v2/pkg/converter/constant"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func createTestImage(width, height int, format string) (image.Image, error) {
func createTestImageFile(t *testing.T, path string, width, height int) {
t.Helper()
img := image.NewRGBA(image.Rect(0, 0, width, height))
// Create a gradient pattern to ensure we have actual image data
for y := 0; y < height; y++ {
for x := 0; x < width; x++ {
img.Set(x, y, color.RGBA{
R: uint8((x * 255) / width),
G: uint8((y * 255) / height),
B: 100,
A: 255,
})
}
}
return img, nil
f, err := os.Create(path)
require.NoError(t, err)
err = jpeg.Encode(f, img, nil)
require.NoError(t, err)
_ = f.Close()
}
func encodeImage(img image.Image, format string) (*bytes.Buffer, string, error) {
buf := new(bytes.Buffer)
func createTestChapter(t *testing.T, pages []struct{ w, h int }) (*manga.Chapter, string) {
t.Helper()
dir := t.TempDir()
inputDir := filepath.Join(dir, "input")
require.NoError(t, os.MkdirAll(inputDir, 0755))
require.NoError(t, os.MkdirAll(filepath.Join(dir, "output"), 0755))
switch format {
case "jpeg", "jpg":
if err := jpeg.Encode(buf, img, &jpeg.Options{Quality: 85}); err != nil {
return nil, "", err
}
return buf, ".jpg", nil
case "gif":
if err := gif.Encode(buf, img, nil); err != nil {
return nil, "", err
}
return buf, ".gif", nil
case "webp":
if err := PrepareEncoder(); err != nil {
return nil, "", err
}
if err := Encode(buf, img, 80); err != nil {
return nil, "", err
}
return buf, ".webp", nil
case "png":
fallthrough
default:
if err := png.Encode(buf, img); err != nil {
return nil, "", err
}
return buf, ".png", nil
}
}
func createTestPage(t *testing.T, index int, width, height int, format string) *manga.Page {
img, err := createTestImage(width, height, format)
require.NoError(t, err)
buf, ext, err := encodeImage(img, format)
require.NoError(t, err)
return &manga.Page{
Index: uint16(index),
Contents: buf,
Extension: ext,
Size: uint64(buf.Len()),
}
}
func validateConvertedImage(t *testing.T, page *manga.Page) {
require.True(t, page.Contents != nil || page.TempFilePath != "", "page should have contents in memory or staged on disk")
require.Greater(t, page.Size, uint64(0))
reader, err := page.Open()
require.NoError(t, err)
defer reader.Close()
data, err := io.ReadAll(reader)
require.NoError(t, err)
require.Greater(t, len(data), 0)
// Try to decode the image
img, format, err := image.Decode(bytes.NewReader(data))
require.NoError(t, err, "Failed to decode converted image")
if page.Extension == ".webp" {
assert.Equal(t, "webp", format, "Expected WebP format")
chapter := &manga.Chapter{
FilePath: filepath.Join(dir, "test.cbz"),
TempDir: dir,
}
require.NotNil(t, img)
bounds := img.Bounds()
assert.Greater(t, bounds.Dx(), 0, "Image width should be positive")
assert.Greater(t, bounds.Dy(), 0, "Image height should be positive")
for i, p := range pages {
pagePath := filepath.Join(inputDir, fmt.Sprintf("%04d.jpg", i))
createTestImageFile(t, pagePath, p.w, p.h)
chapter.Pages = append(chapter.Pages, &manga.PageFile{
Index: uint16(i),
Extension: ".jpg",
FilePath: pagePath,
})
}
return chapter, dir
}
// TestConverter_ConvertChapter tests the ConvertChapter method of the WebP converter.
// It verifies various scenarios including:
// - Converting single normal images
// - Converting multiple normal images
// - Converting tall images with split enabled
// - Handling tall images that exceed maximum height
//
// For each test case it validates:
// - Proper error handling
// - Expected number of output pages
// - Correct page ordering
// - Split page handling and indexing
// - Progress callback behavior
//
// The test uses different image dimensions and split settings to ensure
// the converter handles all cases correctly while maintaining proper
// progress reporting and page ordering.
func TestConverter_ConvertChapter(t *testing.T) {
tests := []struct {
name string
pages []*manga.Page
pages []struct{ w, h int }
split bool
expectSplit bool
expectError bool
@@ -140,47 +63,33 @@ func TestConverter_ConvertChapter(t *testing.T) {
}{
{
name: "Single normal image",
pages: []*manga.Page{createTestPage(t, 1, 800, 1200, "jpeg")},
pages: []struct{ w, h int }{{800, 1200}},
split: false,
expectSplit: false,
numExpected: 1,
},
{
name: "Multiple normal images",
pages: []*manga.Page{
createTestPage(t, 1, 800, 1200, "png"),
createTestPage(t, 2, 800, 1200, "jpeg"),
createTestPage(t, 3, 800, 1200, "gif"),
pages: []struct{ w, h int }{
{800, 1200},
{800, 1200},
{800, 1200},
},
split: false,
expectSplit: false,
numExpected: 3,
},
{
name: "Multiple normal images with webp",
pages: []*manga.Page{
createTestPage(t, 1, 800, 1200, "png"),
createTestPage(t, 2, 800, 1200, "jpeg"),
createTestPage(t, 3, 800, 1200, "gif"),
createTestPage(t, 4, 800, 1200, "webp"),
},
split: false,
expectSplit: false,
numExpected: 4,
},
{
name: "Tall image with split enabled",
pages: []*manga.Page{createTestPage(t, 1, 800, 5000, "jpeg")},
pages: []struct{ w, h int }{{800, 5000}},
split: true,
expectSplit: true,
numExpected: 3, // Based on cropHeight of 2000
expectSplit: false, // cwebp handles 5000px fine (< 16383 webp max), no split needed
numExpected: 1,
},
{
name: "Tall image without split",
pages: []*manga.Page{createTestPage(t, 1, 800, webpMaxHeight+100, "png")},
pages: []struct{ w, h int }{{800, webpMaxHeight + 100}},
split: false,
expectError: true,
numExpected: 1,
numExpected: 1, // kept as-is
},
}
@@ -190,18 +99,16 @@ func TestConverter_ConvertChapter(t *testing.T) {
err := converter.PrepareConverter()
require.NoError(t, err)
chapter := &manga.Chapter{
Pages: tt.pages,
}
chapter, _ := createTestChapter(t, tt.pages)
var progressMutex sync.Mutex
var lastProgress uint32
progress := func(message string, current uint32, total uint32) {
progressMutex.Lock()
defer progressMutex.Unlock()
assert.GreaterOrEqual(t, current, lastProgress, "Progress should never decrease")
assert.GreaterOrEqual(t, current, lastProgress)
lastProgress = current
assert.LessOrEqual(t, current, total, "Current progress should not exceed total")
assert.LessOrEqual(t, current, total)
}
convertedChapter, err := converter.ConvertChapter(context.Background(), chapter, 80, tt.split, progress)
@@ -218,22 +125,14 @@ func TestConverter_ConvertChapter(t *testing.T) {
require.NotNil(t, convertedChapter)
assert.Len(t, convertedChapter.Pages, tt.numExpected)
// Validate all converted images
for _, page := range convertedChapter.Pages {
validateConvertedImage(t, page)
}
// Verify page order
for i := 1; i < len(convertedChapter.Pages); i++ {
prevPage := convertedChapter.Pages[i-1]
currPage := convertedChapter.Pages[i]
if prevPage.Index == currPage.Index {
assert.Less(t, prevPage.SplitPartIndex, currPage.SplitPartIndex,
"Split parts should be in ascending order for page %d", prevPage.Index)
prev := convertedChapter.Pages[i-1]
curr := convertedChapter.Pages[i]
if prev.Index == curr.Index {
assert.Less(t, prev.SplitPartIndex, curr.SplitPartIndex)
} else {
assert.Less(t, prevPage.Index, currPage.Index,
"Pages should be in ascending order")
assert.Less(t, prev.Index, curr.Index)
}
}
@@ -245,149 +144,13 @@ func TestConverter_ConvertChapter(t *testing.T) {
break
}
}
assert.True(t, splitFound, "Expected to find at least one split page")
}
})
}
}
func TestConverter_convertPage(t *testing.T) {
converter := New()
err := converter.PrepareConverter()
require.NoError(t, err)
tests := []struct {
name string
format string
isToBeConverted bool
expectWebP bool
expectError bool
}{
{
name: "Convert PNG to WebP",
format: "png",
isToBeConverted: true,
expectWebP: true,
expectError: false,
},
{
name: "Convert GIF to WebP",
format: "gif",
isToBeConverted: true,
expectWebP: true,
expectError: false,
},
{
name: "Already WebP",
format: "webp",
isToBeConverted: true,
expectWebP: true,
expectError: false,
},
{
name: "Skip conversion",
format: "png",
isToBeConverted: false,
expectWebP: false,
expectError: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
page := createTestPage(t, 1, 100, 100, tt.format)
img, err := createTestImage(100, 100, tt.format)
require.NoError(t, err)
container := manga.NewContainer(page, img, tt.format, tt.isToBeConverted)
converted, err := converter.convertPage(container, 80, t.TempDir())
if tt.expectError {
assert.Error(t, err)
assert.Nil(t, converted)
} else {
require.NoError(t, err)
assert.NotNil(t, converted)
if tt.expectWebP {
assert.Equal(t, ".webp", converted.Page.Extension)
validateConvertedImage(t, converted.Page)
} else {
assert.NotEqual(t, ".webp", converted.Page.Extension)
}
}
})
}
}
func TestConverter_convertPage_EncodingError(t *testing.T) {
converter := New()
err := converter.PrepareConverter()
require.NoError(t, err)
// Create a test case with nil image to test encoding error path
// when isToBeConverted is true but the image is nil, simulating a failure in the encoding step
corruptedPage := &manga.Page{
Index: 1,
Contents: &bytes.Buffer{}, // Empty buffer
Extension: ".png",
Size: 0,
}
container := manga.NewContainer(corruptedPage, nil, "png", true)
converted, err := converter.convertPage(container, 80, t.TempDir())
// This should return nil container and error because encoding will fail with nil image
assert.Error(t, err)
assert.Nil(t, converted)
}
func TestConverter_checkPageNeedsSplit(t *testing.T) {
converter := New()
tests := []struct {
name string
imageHeight int
split bool
expectSplit bool
expectError bool
}{
{
name: "Normal height",
imageHeight: 1000,
split: true,
expectSplit: false,
},
{
name: "Height exceeds max with split enabled",
imageHeight: 5000,
split: true,
expectSplit: true,
},
{
name: "Height exceeds webp max without split",
imageHeight: webpMaxHeight + 100,
split: false,
expectError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
page := createTestPage(t, 1, 800, tt.imageHeight, "jpeg")
needsSplit, img, format, err := converter.checkPageNeedsSplit(page, tt.split)
if tt.expectError {
assert.Error(t, err)
return
assert.True(t, splitFound, "Expected split pages")
}
require.NoError(t, err)
assert.NotNil(t, img)
assert.NotEmpty(t, format)
assert.Equal(t, tt.expectSplit, needsSplit)
// Verify all output files exist
for _, page := range convertedChapter.Pages {
assert.FileExists(t, page.FilePath)
}
})
}
}
@@ -402,73 +165,43 @@ func TestConverter_ConvertChapter_Timeout(t *testing.T) {
err := converter.PrepareConverter()
require.NoError(t, err)
// Create a test chapter with a few pages
pages := []*manga.Page{
createTestPage(t, 1, 800, 1200, "jpeg"),
createTestPage(t, 2, 800, 1200, "png"),
createTestPage(t, 3, 800, 1200, "gif"),
}
chapter, _ := createTestChapter(t, []struct{ w, h int }{
{800, 1200},
{800, 1200},
{800, 1200},
})
chapter := &manga.Chapter{
FilePath: "/test/chapter.cbz",
Pages: pages,
}
progress := func(message string, current uint32, total uint32) {}
var progressMutex sync.Mutex
var lastProgress uint32
progress := func(message string, current uint32, total uint32) {
progressMutex.Lock()
defer progressMutex.Unlock()
assert.GreaterOrEqual(t, current, lastProgress, "Progress should never decrease")
lastProgress = current
assert.LessOrEqual(t, current, total, "Current progress should not exceed total")
}
// Test with very short timeout (1 nanosecond)
ctx, cancel := context.WithTimeout(context.Background(), 1)
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
defer cancel()
convertedChapter, err := converter.ConvertChapter(ctx, chapter, 80, false, progress)
// Should return context error due to timeout
assert.Error(t, err)
assert.Nil(t, convertedChapter)
assert.Equal(t, context.DeadlineExceeded, err)
}
// TestConverter_ConvertChapter_ManyPages_NoDeadlock tests that converting chapters with many pages
// does not cause a deadlock. This test reproduces the scenario where processing
// many files with context cancellation could cause "all goroutines are asleep - deadlock!" error.
// The fix ensures that wgConvertedPages.Done() is called when context is cancelled after Add(1).
func TestConverter_ConvertChapter_ManyPages_NoDeadlock(t *testing.T) {
converter := New()
err := converter.PrepareConverter()
require.NoError(t, err)
// Create a chapter with many pages to increase the chance of hitting the race condition
numPages := 50
pages := make([]*manga.Page, numPages)
for i := 0; i < numPages; i++ {
pages[i] = createTestPage(t, i+1, 100, 100, "jpeg")
pages := make([]struct{ w, h int }, 50)
for i := range pages {
pages[i] = struct{ w, h int }{100, 100}
}
chapter := &manga.Chapter{
FilePath: "/test/chapter_many_pages.cbz",
Pages: pages,
}
chapter, _ := createTestChapter(t, pages)
progress := func(message string, current uint32, total uint32) {
// No-op progress callback
}
progress := func(message string, current uint32, total uint32) {}
// Run multiple iterations to increase the chance of hitting the race condition
for iteration := 0; iteration < 10; iteration++ {
t.Run(fmt.Sprintf("iteration_%d", iteration), func(t *testing.T) {
// Use a very short timeout to trigger context cancellation during processing
ctx, cancel := context.WithTimeout(context.Background(), time.Nanosecond)
defer cancel()
// This should NOT deadlock - it should return quickly with context error
done := make(chan struct{})
var convertErr error
go func() {
@@ -476,121 +209,57 @@ func TestConverter_ConvertChapter_ManyPages_NoDeadlock(t *testing.T) {
_, convertErr = converter.ConvertChapter(ctx, chapter, 80, false, progress)
}()
// Wait with a reasonable timeout - if it takes longer than 5 seconds, we have a deadlock
select {
case <-done:
// Expected - conversion should complete (with error) quickly
assert.Error(t, convertErr, "Expected context error")
assert.Error(t, convertErr)
case <-time.After(5 * time.Second):
t.Fatal("Deadlock detected: ConvertChapter did not return within 5 seconds")
t.Fatal("Deadlock detected")
}
})
}
}
// TestConverter_ConvertChapter_ManyPages_WithSplit_NoDeadlock tests that converting chapters
// with many pages and split enabled does not cause a deadlock.
func TestConverter_ConvertChapter_ManyPages_WithSplit_NoDeadlock(t *testing.T) {
converter := New()
err := converter.PrepareConverter()
require.NoError(t, err)
// Create pages with varying heights, some requiring splits
numPages := 30
pages := make([]*manga.Page, numPages)
for i := 0; i < numPages; i++ {
height := 1000 // Normal height
if i%5 == 0 {
height = 5000 // Tall image that will be split
}
pages[i] = createTestPage(t, i+1, 100, height, "png")
}
chapter := &manga.Chapter{
FilePath: "/test/chapter_split_test.cbz",
Pages: pages,
}
progress := func(message string, current uint32, total uint32) {
// No-op progress callback
}
// Run multiple iterations with short timeouts
for iteration := 0; iteration < 10; iteration++ {
t.Run(fmt.Sprintf("iteration_%d", iteration), func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Nanosecond)
defer cancel()
done := make(chan struct{})
var convertErr error
go func() {
defer close(done)
_, convertErr = converter.ConvertChapter(ctx, chapter, 80, true, progress) // split=true
}()
select {
case <-done:
assert.Error(t, convertErr, "Expected context error")
case <-time.After(5 * time.Second):
t.Fatal("Deadlock detected: ConvertChapter with split did not return within 5 seconds")
}
})
}
}
// TestConverter_ConvertChapter_ConcurrentChapters_NoDeadlock simulates the scenario from the
// original bug report where multiple chapters are processed in parallel with parallelism > 1.
// This test ensures no deadlock occurs when multiple goroutines are converting chapters concurrently.
func TestConverter_ConvertChapter_ConcurrentChapters_NoDeadlock(t *testing.T) {
converter := New()
err := converter.PrepareConverter()
require.NoError(t, err)
// Create multiple chapters, each with many pages
numChapters := 20
pagesPerChapter := 30
chapters := make([]*manga.Chapter, numChapters)
pages := make([]struct{ w, h int }, 30)
for i := range pages {
pages[i] = struct{ w, h int }{100, 100}
}
for c := 0; c < numChapters; c++ {
pages := make([]*manga.Page, pagesPerChapter)
for i := 0; i < pagesPerChapter; i++ {
pages[i] = createTestPage(t, i+1, 100, 100, "jpeg")
}
chapters[c] = &manga.Chapter{
FilePath: fmt.Sprintf("/test/chapter_%d.cbz", c+1),
Pages: pages,
}
chapters[c], _ = createTestChapter(t, pages)
}
progress := func(message string, current uint32, total uint32) {}
// Process chapters concurrently with short timeouts (simulating parallelism flag)
parallelism := 4
var wg sync.WaitGroup
semaphore := make(chan struct{}, parallelism)
// Overall test timeout
testCtx, testCancel := context.WithTimeout(context.Background(), 30*time.Second)
defer testCancel()
for _, chapter := range chapters {
wg.Add(1)
semaphore <- struct{}{} // Acquire
semaphore <- struct{}{}
go func(ch *manga.Chapter) {
defer wg.Done()
defer func() { <-semaphore }() // Release
defer func() { <-semaphore }()
// Use very short timeout to trigger cancellation
ctx, cancel := context.WithTimeout(context.Background(), time.Nanosecond)
defer cancel()
// This should not deadlock
_, _ = converter.ConvertChapter(ctx, ch, 80, false, progress)
}(chapter)
}
// Wait for all conversions with a timeout
done := make(chan struct{})
go func() {
wg.Wait()
@@ -599,8 +268,220 @@ func TestConverter_ConvertChapter_ConcurrentChapters_NoDeadlock(t *testing.T) {
select {
case <-done:
// All goroutines completed successfully
case <-testCtx.Done():
t.Fatal("Deadlock detected: Concurrent chapter conversions did not complete within 30 seconds")
t.Fatal("Deadlock detected")
}
}
func TestConverter_SplitAndConvert(t *testing.T) {
converter := New()
err := converter.PrepareConverter()
require.NoError(t, err)
// Create a very tall image that exceeds webpMaxHeight
dir := t.TempDir()
inputDir := filepath.Join(dir, "input")
require.NoError(t, os.MkdirAll(inputDir, 0755))
require.NoError(t, os.MkdirAll(filepath.Join(dir, "output"), 0755))
// Create image taller than webpMaxHeight (16383)
tallImagePath := filepath.Join(inputDir, "0000.jpg")
createTestImageFile(t, tallImagePath, 800, webpMaxHeight+500)
chapter := &manga.Chapter{
FilePath: filepath.Join(dir, "test.cbz"),
TempDir: dir,
Pages: []*manga.PageFile{
{Index: 0, Extension: ".jpg", FilePath: tallImagePath},
},
}
progress := func(message string, current uint32, total uint32) {}
// With split=true, the oversized page should be split
convertedChapter, err := converter.ConvertChapter(context.Background(), chapter, 80, true, progress)
require.NoError(t, err)
require.NotNil(t, convertedChapter)
// Should have more pages due to splitting
assert.Greater(t, len(convertedChapter.Pages), 1, "Tall image should be split into multiple parts")
// All parts should be split and have webp extension
for _, page := range convertedChapter.Pages {
assert.True(t, page.IsSplitted, "All pages should be marked as split")
assert.Equal(t, ".webp", page.Extension)
assert.FileExists(t, page.FilePath)
assert.Equal(t, uint16(0), page.Index, "All split parts should have same original index")
}
// Verify sequential split part indices
for i, page := range convertedChapter.Pages {
assert.Equal(t, uint16(i), page.SplitPartIndex)
}
}
func TestConverter_OversizedImageNoSplit(t *testing.T) {
converter := New()
err := converter.PrepareConverter()
require.NoError(t, err)
dir := t.TempDir()
inputDir := filepath.Join(dir, "input")
require.NoError(t, os.MkdirAll(inputDir, 0755))
require.NoError(t, os.MkdirAll(filepath.Join(dir, "output"), 0755))
tallImagePath := filepath.Join(inputDir, "0000.jpg")
createTestImageFile(t, tallImagePath, 800, webpMaxHeight+500)
chapter := &manga.Chapter{
FilePath: filepath.Join(dir, "test.cbz"),
TempDir: dir,
Pages: []*manga.PageFile{
{Index: 0, Extension: ".jpg", FilePath: tallImagePath},
},
}
progress := func(message string, current uint32, total uint32) {}
// With split=false, the oversized page should be kept as-is with error
convertedChapter, err := converter.ConvertChapter(context.Background(), chapter, 80, false, progress)
assert.Error(t, err, "Should return error for oversized page without split")
if convertedChapter != nil {
// The page should be kept in original format
for _, page := range convertedChapter.Pages {
assert.Equal(t, ".jpg", page.Extension, "Page should keep original extension")
}
}
}
func TestGetImageDimensions(t *testing.T) {
dir := t.TempDir()
tests := []struct {
name string
width, height int
expectW, expectH int
}{
{"small image", 100, 200, 100, 200},
{"wide image", 1920, 1080, 1920, 1080},
{"tall image", 800, 5000, 800, 5000},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
path := filepath.Join(dir, tt.name+".jpg")
createTestImageFile(t, path, tt.width, tt.height)
w, h, err := getImageDimensions(path)
require.NoError(t, err)
assert.Equal(t, tt.expectW, w)
assert.Equal(t, tt.expectH, h)
})
}
}
func TestGetImageDimensions_NonexistentFile(t *testing.T) {
_, _, err := getImageDimensions("/nonexistent/file.jpg")
assert.Error(t, err)
}
func TestGetImageDimensions_InvalidFile(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "invalid.jpg")
require.NoError(t, os.WriteFile(path, []byte("not an image"), 0644))
_, _, err := getImageDimensions(path)
assert.Error(t, err)
}
func TestConverter_ConvertChapter_EmptyChapter(t *testing.T) {
converter := New()
err := converter.PrepareConverter()
require.NoError(t, err)
dir := t.TempDir()
chapter := &manga.Chapter{
FilePath: filepath.Join(dir, "test.cbz"),
TempDir: dir,
Pages: []*manga.PageFile{},
}
progress := func(message string, current uint32, total uint32) {}
_, err = converter.ConvertChapter(context.Background(), chapter, 80, false, progress)
assert.Error(t, err, "Should error on empty chapter")
}
func TestConverter_ConvertChapter_PreservesComicInfo(t *testing.T) {
converter := New()
err := converter.PrepareConverter()
require.NoError(t, err)
chapter, _ := createTestChapter(t, []struct{ w, h int }{{400, 600}})
chapter.ComicInfoXml = `<?xml version="1.0"?><ComicInfo><Series>Test</Series></ComicInfo>`
progress := func(message string, current uint32, total uint32) {}
convertedChapter, err := converter.ConvertChapter(context.Background(), chapter, 80, false, progress)
require.NoError(t, err)
require.NotNil(t, convertedChapter)
assert.Equal(t, chapter.ComicInfoXml, convertedChapter.ComicInfoXml)
}
func TestConverter_ConvertChapter_OutputInCorrectDir(t *testing.T) {
converter := New()
err := converter.PrepareConverter()
require.NoError(t, err)
chapter, dir := createTestChapter(t, []struct{ w, h int }{{400, 600}})
progress := func(message string, current uint32, total uint32) {}
convertedChapter, err := converter.ConvertChapter(context.Background(), chapter, 80, false, progress)
require.NoError(t, err)
require.NotNil(t, convertedChapter)
// All output files should be in the output directory
expectedOutputDir := filepath.Join(dir, "output")
for _, page := range convertedChapter.Pages {
pageDir := filepath.Dir(page.FilePath)
assert.Equal(t, expectedOutputDir, pageDir, "Output file should be in output directory")
}
}
func TestEncodeFile(t *testing.T) {
err := PrepareEncoder()
require.NoError(t, err)
dir := t.TempDir()
inputPath := filepath.Join(dir, "input.jpg")
outputPath := filepath.Join(dir, "output.webp")
createTestImageFile(t, inputPath, 200, 300)
err = EncodeFile(inputPath, outputPath, 80)
require.NoError(t, err)
// Verify output exists and is non-empty
info, err := os.Stat(outputPath)
require.NoError(t, err)
assert.Greater(t, info.Size(), int64(0))
}
func TestEncodeFileWithCrop(t *testing.T) {
err := PrepareEncoder()
require.NoError(t, err)
dir := t.TempDir()
inputPath := filepath.Join(dir, "input.jpg")
outputPath := filepath.Join(dir, "output.webp")
createTestImageFile(t, inputPath, 200, 600)
err = EncodeFileWithCrop(inputPath, outputPath, 80, 0, 0, 200, 300)
require.NoError(t, err)
info, err := os.Stat(outputPath)
require.NoError(t, err)
assert.Greater(t, info.Size(), int64(0))
}
+16 -5
View File
@@ -2,8 +2,6 @@ package webp
import (
"fmt"
"image"
"io"
"strings"
"sync"
@@ -37,10 +35,23 @@ func PrepareEncoder() error {
return nil
}
func Encode(w io.Writer, m image.Image, quality uint) error {
// EncodeFile converts an image file directly to WebP using cwebp.
// This is a zero-copy operation: no image data is loaded into Go memory.
func EncodeFile(inputPath string, outputPath string, quality uint) error {
return webpbin.NewCWebP(config).
Quality(quality).
InputImage(m).
Output(w).
InputFile(inputPath).
OutputFile(outputPath).
Run()
}
// EncodeFileWithCrop converts a cropped region of an image file to WebP.
// Uses cwebp's native -crop flag — no Go-side image decode needed.
func EncodeFileWithCrop(inputPath string, outputPath string, quality uint, x, y, width, height int) error {
return webpbin.NewCWebP(config).
Quality(quality).
Crop(x, y, width, height).
InputFile(inputPath).
OutputFile(outputPath).
Run()
}