fix(memory): use staging folder to avoid keeping everything in memory

Co-authored-by: Belphemur <197810+Belphemur@users.noreply.github.com> Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Fixes #203
2026-07-21 19:05:39 +02:00 · 2026-07-03 09:15:56 -04:00
parent f2a65425cf
commit c2f809ed2c
13 changed files with 352 additions and 33 deletions
@@ -9,6 +9,7 @@ import (
 	_ "image/gif"
 	_ "image/jpeg"
 	"image/png"
+	"os"
 	"runtime"
 	"sync"
 	"sync/atomic"
@@ -28,6 +29,15 @@ type Converter struct {
 	maxHeight  int
 	cropHeight int
 	isPrepared bool
+	// pageWorkerGuard limits how many pages are converted (i.e. how many
+	// cwebp binary processes are spawned) concurrently across the whole
+	// application, regardless of how many chapters are being processed in
+	// parallel (see the --parallelism flag). Without this shared limit, the
+	// per-chapter worker pool (sized to runtime.NumCPU()) would be
+	// multiplied by the number of chapters processed concurrently, leading
+	// to CPU/memory oversubscription since every page conversion spawns an
+	// external cwebp process.
+	pageWorkerGuard chan struct{}
 }

 func (converter *Converter) Format() (format constant.ConversionFormat) {
@@ -37,9 +47,10 @@ func (converter *Converter) Format() (format constant.ConversionFormat) {
 func New() *Converter {
 	return &Converter{
 		//maxHeight: 16383 / 2,
-		maxHeight:  4000,
-		cropHeight: 2000,
-		isPrepared: false,
+		maxHeight:       4000,
+		cropHeight:      2000,
+		isPrepared:      false,
+		pageWorkerGuard: make(chan struct{}, runtime.NumCPU()),
 	}
 }

@@ -70,6 +81,18 @@ func (converter *Converter) ConvertChapter(ctx context.Context, chapter *manga.C
 		return nil, err
 	}

+	// Stage converted pages on disk in a temp folder instead of keeping them
+	// all in memory until the whole chapter is written out. This bounds
+	// memory usage for chapters with many/large pages; the caller is
+	// responsible for calling chapter.Cleanup() once the chapter has been
+	// written to its final CBZ file.
+	tempDir, err := os.MkdirTemp("", "cbzoptimizer-*")
+	if err != nil {
+		log.Error().Str("chapter", chapter.FilePath).Err(err).Msg("Failed to create staging temp folder")
+		return nil, fmt.Errorf("failed to create staging temp folder: %w", err)
+	}
+	chapter.TempDir = tempDir
+
 	var wgConvertedPages sync.WaitGroup
 	maxGoroutines := runtime.NumCPU()

@@ -80,7 +103,12 @@ func (converter *Converter) ConvertChapter(ctx context.Context, chapter *manga.C
 	var wgPages sync.WaitGroup
 	wgPages.Add(len(chapter.Pages))

-	guard := make(chan struct{}, maxGoroutines)
+	// guard is shared across all chapters processed concurrently by this
+	// converter instance (see pageWorkerGuard doc comment) so that the total
+	// number of concurrently running cwebp processes stays bounded to
+	// runtime.NumCPU(), no matter how many chapters are being converted in
+	// parallel at the same time.
+	guard := converter.pageWorkerGuard
 	pagesMutex := sync.Mutex{}
 	var pages []*manga.Page
 	var totalPages = uint32(len(chapter.Pages))
@@ -89,13 +117,25 @@ func (converter *Converter) ConvertChapter(ctx context.Context, chapter *manga.C
 		Str("chapter", chapter.FilePath).
 		Int("total_pages", len(chapter.Pages)).
 		Int("worker_count", maxGoroutines).
+		Str("staging_dir", tempDir).
 		Msg("Initialized conversion worker pool")

+	// failWithCleanup removes the staging temp folder before returning a
+	// fatal error for which no chapter is returned to the caller (and thus
+	// nobody else will clean it up).
+	failWithCleanup := func(err error) (*manga.Chapter, error) {
+		if removeErr := os.RemoveAll(tempDir); removeErr != nil {
+			log.Warn().Str("chapter", chapter.FilePath).Err(removeErr).Msg("Failed to remove staging temp folder after error")
+		}
+		chapter.TempDir = ""
+		return nil, err
+	}
+
 	// Check if context is already cancelled
 	select {
 	case <-ctx.Done():
 		log.Warn().Str("chapter", chapter.FilePath).Msg("Chapter conversion cancelled due to timeout")
-		return nil, ctx.Err()
+		return failWithCleanup(ctx.Err())
 	default:
 	}

@@ -122,7 +162,7 @@ func (converter *Converter) ConvertChapter(ctx context.Context, chapter *manga.C
 				default:
 				}

-				convertedPage, err := converter.convertPage(pageToConvert, quality)
+				convertedPage, err := converter.convertPage(pageToConvert, quality, tempDir)
 				if err != nil {
 					if convertedPage == nil {
 						select {
@@ -142,9 +182,14 @@ func (converter *Converter) ConvertChapter(ctx context.Context, chapter *manga.C
 						}
 						return
 					}
-					convertedPage.Page.Contents = buffer
-					convertedPage.Page.Extension = ".png"
-					convertedPage.Page.Size = uint64(buffer.Len())
+					if err := convertedPage.Page.Stage(tempDir, buffer, ".png"); err != nil {
+						select {
+						case errChan <- err:
+						case <-ctx.Done():
+							return
+						}
+						return
+					}
 				}
 				pagesMutex.Lock()
 				defer pagesMutex.Unlock()
@@ -160,7 +205,7 @@ func (converter *Converter) ConvertChapter(ctx context.Context, chapter *manga.C
 		select {
 		case <-ctx.Done():
 			log.Warn().Str("chapter", chapter.FilePath).Msg("Chapter conversion cancelled due to timeout")
-			return nil, ctx.Err()
+			return failWithCleanup(ctx.Err())
 		default:
 		}

@@ -251,11 +296,13 @@ func (converter *Converter) ConvertChapter(ctx context.Context, chapter *manga.C
 		// Conversion completed successfully
 	case <-ctx.Done():
 		log.Warn().Str("chapter", chapter.FilePath).Msg("Chapter conversion cancelled due to timeout")
-		return nil, ctx.Err()
+		return failWithCleanup(ctx.Err())
 	}

 	close(errChan)
-	close(guard)
+	// Note: guard is the converter's shared pageWorkerGuard and must not be
+	// closed here since it is reused by subsequent (and possibly concurrent)
+	// ConvertChapter calls.

 	var errList []error
 	for err := range errChan {
@@ -402,7 +449,7 @@ func (converter *Converter) checkPageNeedsSplit(page *manga.Page, splitRequested
 	return needsSplit, img, format, nil
 }

-func (converter *Converter) convertPage(container *manga.PageContainer, quality uint8) (*manga.PageContainer, error) {
+func (converter *Converter) convertPage(container *manga.PageContainer, quality uint8, tempDir string) (*manga.PageContainer, error) {
 	log.Debug().
 		Uint16("page_index", container.Page.Index).
 		Str("format", container.Format).
@@ -439,12 +486,22 @@ func (converter *Converter) convertPage(container *manga.PageContainer, quality
 		return nil, err
 	}

-	container.SetConverted(converted, ".webp")
+	originalSize := container.Page.Size
+	convertedSize := converted.Len()
+
+	if err := container.Page.Stage(tempDir, converted, ".webp"); err != nil {
+		log.Error().
+			Uint16("page_index", container.Page.Index).
+			Err(err).
+			Msg("Failed to stage converted page to disk")
+		return nil, err
+	}
+	container.HasBeenConverted = true

 	log.Debug().
 		Uint16("page_index", container.Page.Index).
-		Int("original_size", len(container.Page.Contents.Bytes())).
-		Int("converted_size", len(converted.Bytes())).
+		Uint64("original_size", originalSize).
+		Int("converted_size", convertedSize).
 		Msg("Page conversion completed")

 	return container, nil
@@ -9,6 +9,7 @@ import (
 	"image/gif"
 	"image/jpeg"
 	"image/png"
+	"io"
 	"sync"
 	"testing"
 	"time"
@@ -86,11 +87,19 @@ func createTestPage(t *testing.T, index int, width, height int, format string) *
 }

 func validateConvertedImage(t *testing.T, page *manga.Page) {
-	require.NotNil(t, page.Contents)
-	require.Greater(t, page.Contents.Len(), 0)
+	require.True(t, page.Contents != nil || page.TempFilePath != "", "page should have contents in memory or staged on disk")
+	require.Greater(t, page.Size, uint64(0))
+
+	reader, err := page.Open()
+	require.NoError(t, err)
+	defer reader.Close()
+
+	data, err := io.ReadAll(reader)
+	require.NoError(t, err)
+	require.Greater(t, len(data), 0)

 	// Try to decode the image
-	img, format, err := image.Decode(bytes.NewReader(page.Contents.Bytes()))
+	img, format, err := image.Decode(bytes.NewReader(data))
 	require.NoError(t, err, "Failed to decode converted image")

 	if page.Extension == ".webp" {
@@ -291,7 +300,7 @@ func TestConverter_convertPage(t *testing.T) {
 			require.NoError(t, err)
 			container := manga.NewContainer(page, img, tt.format, tt.isToBeConverted)

-			converted, err := converter.convertPage(container, 80)
+			converted, err := converter.convertPage(container, 80, t.TempDir())

 			if tt.expectError {
 				assert.Error(t, err)
@@ -327,7 +336,7 @@ func TestConverter_convertPage_EncodingError(t *testing.T) {

 	container := manga.NewContainer(corruptedPage, nil, "png", true)

-	converted, err := converter.convertPage(container, 80)
+	converted, err := converter.convertPage(container, 80, t.TempDir())

 	// This should return nil container and error because encoding will fail with nil image
 	assert.Error(t, err)