mirror of
https://github.com/Belphemur/CBZOptimizer.git
synced 2026-07-21 19:05:39 +02:00
refactor!: disk-first zero-copy architecture with file-to-file cwebp conversion
BREAKING CHANGE: Complete refactoring of the conversion pipeline. - Replace in-memory Page/PageContainer with disk-based PageFile struct - Extract archives to temp directory instead of loading into memory - Use cwebp InputFile/OutputFile for zero-copy file-to-file conversion - Use cwebp -crop for splitting (no Go-side image decode needed) - Check if already converted before extracting (fast pre-check) - Remove oliamb/cutter dependency (replaced by cwebp -crop) - Remove page_container.go (no longer needed) - Add comprehensive tests with improved coverage Co-authored-by: Belphemur <197810+Belphemur@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
co-authored by
Belphemur
Belphemur
parent
e87e73eb28
commit
164dc577b9
+136
-173
@@ -1,24 +1,23 @@
|
||||
package converter
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/belphemur/CBZOptimizer/v2/internal/manga"
|
||||
"github.com/belphemur/CBZOptimizer/v2/internal/utils/errs"
|
||||
)
|
||||
|
||||
func TestConvertChapter(t *testing.T) {
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
genTestChapter func(path string, isSplit bool) (*manga.Chapter, []string, error)
|
||||
split bool
|
||||
expectError bool
|
||||
name string
|
||||
genTestChapter func(t *testing.T, dir string) (*manga.Chapter, []string)
|
||||
split bool
|
||||
expectError bool
|
||||
}{
|
||||
{
|
||||
name: "All split pages",
|
||||
@@ -27,7 +26,7 @@ func TestConvertChapter(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "Big Pages, no split",
|
||||
genTestChapter: genHugePage,
|
||||
genTestChapter: genHugePageNoSplit,
|
||||
split: false,
|
||||
expectError: true,
|
||||
},
|
||||
@@ -47,53 +46,41 @@ func TestConvertChapter(t *testing.T) {
|
||||
split: false,
|
||||
expectError: true,
|
||||
},
|
||||
{
|
||||
name: "Two corrupted pages",
|
||||
genTestChapter: genTwoCorrupted,
|
||||
split: false,
|
||||
expectError: true,
|
||||
},
|
||||
}
|
||||
// Load test genTestChapter from testdata
|
||||
temp, err := os.CreateTemp("", "test_chapter_*.cbz")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create temporary file: %v", err)
|
||||
|
||||
}
|
||||
defer errs.CaptureGeneric(&err, os.Remove, temp.Name(), "failed to remove temporary file")
|
||||
for _, converter := range Available() {
|
||||
converter, err := Get(converter)
|
||||
for _, converterFormat := range Available() {
|
||||
conv, err := Get(converterFormat)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get converter: %v", err)
|
||||
}
|
||||
t.Run(converter.Format().String(), func(t *testing.T) {
|
||||
t.Run(conv.Format().String(), func(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
chapter, expectedExtensions, err := tc.genTestChapter(temp.Name(), tc.split)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to load test genTestChapter: %v", err)
|
||||
}
|
||||
tempDir := t.TempDir()
|
||||
chapter, expectedExtensions := tc.genTestChapter(t, tempDir)
|
||||
|
||||
quality := uint8(80)
|
||||
|
||||
progress := func(msg string, current uint32, total uint32) {
|
||||
t.Log(msg)
|
||||
}
|
||||
|
||||
convertedChapter, err := converter.ConvertChapter(context.Background(), chapter, quality, tc.split, progress)
|
||||
convertedChapter, err := conv.ConvertChapter(context.Background(), chapter, quality, tc.split, progress)
|
||||
if err != nil && !tc.expectError {
|
||||
t.Fatalf("failed to convert genTestChapter: %v", err)
|
||||
t.Fatalf("failed to convert chapter: %v", err)
|
||||
}
|
||||
|
||||
if convertedChapter == nil {
|
||||
t.Fatal("convertedChapter is nil")
|
||||
}
|
||||
|
||||
if len(convertedChapter.Pages) == 0 {
|
||||
t.Fatalf("no pages were converted")
|
||||
t.Fatal("no pages were converted")
|
||||
}
|
||||
|
||||
if len(convertedChapter.Pages) != len(expectedExtensions) {
|
||||
t.Fatalf("converted chapter has %d pages but expected %d", len(convertedChapter.Pages), len(expectedExtensions))
|
||||
}
|
||||
|
||||
// Check each page's extension against the expected array
|
||||
for i, page := range convertedChapter.Pages {
|
||||
expectedExt := expectedExtensions[i]
|
||||
if page.Extension != expectedExt {
|
||||
@@ -106,176 +93,152 @@ func TestConvertChapter(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func genHugePage(path string, isSplit bool) (*manga.Chapter, []string, error) {
|
||||
file, err := os.Open(path)
|
||||
// createTestImageFile creates a JPEG image file at the given path with the specified dimensions.
|
||||
func createTestImageFile(t *testing.T, path string, width, height int) {
|
||||
t.Helper()
|
||||
img := image.NewRGBA(image.Rect(0, 0, width, height))
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer errs.Capture(&err, file.Close, "failed to close file")
|
||||
|
||||
var pages []*manga.Page
|
||||
expectedExtensions := []string{".jpg"} // One image that's generated as JPEG
|
||||
if isSplit {
|
||||
expectedExtensions = []string{".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".webp"}
|
||||
}
|
||||
|
||||
// Create one tall page
|
||||
img := image.NewRGBA(image.Rect(0, 0, 1, 17000))
|
||||
buf := new(bytes.Buffer)
|
||||
err = jpeg.Encode(buf, img, nil)
|
||||
err = jpeg.Encode(f, img, nil)
|
||||
_ = f.Close()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
t.Fatal(err)
|
||||
}
|
||||
page := &manga.Page{
|
||||
Index: 0,
|
||||
Contents: buf,
|
||||
Extension: ".jpg",
|
||||
}
|
||||
pages = append(pages, page)
|
||||
|
||||
return &manga.Chapter{
|
||||
FilePath: path,
|
||||
Pages: pages,
|
||||
}, expectedExtensions, nil
|
||||
}
|
||||
|
||||
func genSmallPages(path string, isSplit bool) (*manga.Chapter, []string, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
func genHugePage(t *testing.T, dir string) (*manga.Chapter, []string) {
|
||||
inputDir := filepath.Join(dir, "input")
|
||||
if err := os.MkdirAll(inputDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(dir, "output"), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer errs.Capture(&err, file.Close, "failed to close file")
|
||||
|
||||
var pages []*manga.Page
|
||||
for i := 0; i < 5; i++ { // Assuming there are 5 pages for the test
|
||||
img := image.NewRGBA(image.Rect(0, 0, 300, 1000))
|
||||
buf := new(bytes.Buffer)
|
||||
err = jpeg.Encode(buf, img, nil)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
page := &manga.Page{
|
||||
pagePath := filepath.Join(inputDir, "0000.jpg")
|
||||
createTestImageFile(t, pagePath, 1, 17000)
|
||||
|
||||
chapter := &manga.Chapter{
|
||||
FilePath: filepath.Join(dir, "test.cbz"),
|
||||
TempDir: dir,
|
||||
Pages: []*manga.PageFile{
|
||||
{Index: 0, Extension: ".jpg", FilePath: pagePath},
|
||||
},
|
||||
}
|
||||
|
||||
// With split: 17000/2000 = 9 parts (8*2000 + 1*1000)
|
||||
// Without split: page > webpMaxHeight → kept as .jpg with error
|
||||
// The caller decides split=true or split=false and we return expectations
|
||||
// for the split=true case (this test case is always called with split=true)
|
||||
expectedExtensions := []string{".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".webp"}
|
||||
|
||||
return chapter, expectedExtensions
|
||||
}
|
||||
|
||||
func genHugePageNoSplit(t *testing.T, dir string) (*manga.Chapter, []string) {
|
||||
inputDir := filepath.Join(dir, "input")
|
||||
if err := os.MkdirAll(inputDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(dir, "output"), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
pagePath := filepath.Join(inputDir, "0000.jpg")
|
||||
createTestImageFile(t, pagePath, 1, 17000)
|
||||
|
||||
chapter := &manga.Chapter{
|
||||
FilePath: filepath.Join(dir, "test.cbz"),
|
||||
TempDir: dir,
|
||||
Pages: []*manga.PageFile{
|
||||
{Index: 0, Extension: ".jpg", FilePath: pagePath},
|
||||
},
|
||||
}
|
||||
|
||||
// Without split: page > webpMaxHeight → kept as .jpg with error
|
||||
return chapter, []string{".jpg"}
|
||||
}
|
||||
|
||||
func genSmallPages(t *testing.T, dir string) (*manga.Chapter, []string) {
|
||||
inputDir := filepath.Join(dir, "input")
|
||||
_ = os.MkdirAll(inputDir, 0755)
|
||||
_ = os.MkdirAll(filepath.Join(dir, "output"), 0755)
|
||||
|
||||
var pages []*manga.PageFile
|
||||
for i := 0; i < 5; i++ {
|
||||
pagePath := filepath.Join(inputDir, fmt.Sprintf("%04d.jpg", i))
|
||||
createTestImageFile(t, pagePath, 300, 1000)
|
||||
pages = append(pages, &manga.PageFile{
|
||||
Index: uint16(i),
|
||||
Contents: buf,
|
||||
Extension: ".jpg",
|
||||
}
|
||||
pages = append(pages, page)
|
||||
FilePath: pagePath,
|
||||
})
|
||||
}
|
||||
|
||||
return &manga.Chapter{
|
||||
FilePath: path,
|
||||
chapter := &manga.Chapter{
|
||||
FilePath: filepath.Join(dir, "test.cbz"),
|
||||
TempDir: dir,
|
||||
Pages: pages,
|
||||
}, []string{".webp", ".webp", ".webp", ".webp", ".webp"}, nil
|
||||
}
|
||||
|
||||
return chapter, []string{".webp", ".webp", ".webp", ".webp", ".webp"}
|
||||
}
|
||||
|
||||
func genMixSmallBig(path string, isSplit bool) (*manga.Chapter, []string, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer errs.Capture(&err, file.Close, "failed to close file")
|
||||
func genMixSmallBig(t *testing.T, dir string) (*manga.Chapter, []string) {
|
||||
inputDir := filepath.Join(dir, "input")
|
||||
_ = os.MkdirAll(inputDir, 0755)
|
||||
_ = os.MkdirAll(filepath.Join(dir, "output"), 0755)
|
||||
|
||||
var pages []*manga.Page
|
||||
for i := 0; i < 5; i++ { // Assuming there are 5 pages for the test
|
||||
img := image.NewRGBA(image.Rect(0, 0, 300, 1000*(i+1)))
|
||||
buf := new(bytes.Buffer)
|
||||
err := jpeg.Encode(buf, img, nil)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
page := &manga.Page{
|
||||
var pages []*manga.PageFile
|
||||
for i := 0; i < 5; i++ {
|
||||
pagePath := filepath.Join(inputDir, fmt.Sprintf("%04d.jpg", i))
|
||||
createTestImageFile(t, pagePath, 300, 1000*(i+1))
|
||||
pages = append(pages, &manga.PageFile{
|
||||
Index: uint16(i),
|
||||
Contents: buf,
|
||||
Extension: ".jpg",
|
||||
}
|
||||
pages = append(pages, page)
|
||||
}
|
||||
expectedExtensions := []string{".webp", ".webp", ".webp", ".webp", ".webp"}
|
||||
if isSplit {
|
||||
expectedExtensions = []string{".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".webp"}
|
||||
FilePath: pagePath,
|
||||
})
|
||||
}
|
||||
|
||||
return &manga.Chapter{
|
||||
FilePath: path,
|
||||
chapter := &manga.Chapter{
|
||||
FilePath: filepath.Join(dir, "test.cbz"),
|
||||
TempDir: dir,
|
||||
Pages: pages,
|
||||
}, expectedExtensions, nil
|
||||
}
|
||||
|
||||
// Pages heights: 1000, 2000, 3000, 4000, 5000
|
||||
// With new disk-first architecture: cwebp handles all these directly
|
||||
// (all < 16383 webp max), no splitting needed
|
||||
return chapter, []string{".webp", ".webp", ".webp", ".webp", ".webp"}
|
||||
}
|
||||
|
||||
func genMixSmallHuge(path string, isSplit bool) (*manga.Chapter, []string, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer errs.Capture(&err, file.Close, "failed to close file")
|
||||
func genMixSmallHuge(t *testing.T, dir string) (*manga.Chapter, []string) {
|
||||
inputDir := filepath.Join(dir, "input")
|
||||
_ = os.MkdirAll(inputDir, 0755)
|
||||
_ = os.MkdirAll(filepath.Join(dir, "output"), 0755)
|
||||
|
||||
var pages []*manga.Page
|
||||
for i := 0; i < 10; i++ { // Assuming there are 5 pages for the test
|
||||
img := image.NewRGBA(image.Rect(0, 0, 1, 2000*(i+1)))
|
||||
buf := new(bytes.Buffer)
|
||||
err := jpeg.Encode(buf, img, nil)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
page := &manga.Page{
|
||||
var pages []*manga.PageFile
|
||||
for i := 0; i < 10; i++ {
|
||||
pagePath := filepath.Join(inputDir, fmt.Sprintf("%04d.jpg", i))
|
||||
createTestImageFile(t, pagePath, 1, 2000*(i+1))
|
||||
pages = append(pages, &manga.PageFile{
|
||||
Index: uint16(i),
|
||||
Contents: buf,
|
||||
Extension: ".jpg",
|
||||
}
|
||||
pages = append(pages, page)
|
||||
FilePath: pagePath,
|
||||
})
|
||||
}
|
||||
|
||||
return &manga.Chapter{
|
||||
FilePath: path,
|
||||
chapter := &manga.Chapter{
|
||||
FilePath: filepath.Join(dir, "test.cbz"),
|
||||
TempDir: dir,
|
||||
Pages: pages,
|
||||
}, []string{".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".jpg", ".jpg"}, nil
|
||||
}
|
||||
|
||||
func genTwoCorrupted(path string, isSplit bool) (*manga.Chapter, []string, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer errs.Capture(&err, file.Close, "failed to close file")
|
||||
|
||||
var pages []*manga.Page
|
||||
numPages := 5
|
||||
corruptedIndices := []int{2, 4} // Pages 2 and 4 are too tall to convert without splitting
|
||||
for i := 0; i < numPages; i++ {
|
||||
var buf *bytes.Buffer
|
||||
var ext string
|
||||
isCorrupted := false
|
||||
for _, ci := range corruptedIndices {
|
||||
if i == ci {
|
||||
isCorrupted = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if isCorrupted {
|
||||
buf = bytes.NewBufferString("corrupted data") // Invalid data, can't decode as image
|
||||
ext = ".jpg"
|
||||
} else {
|
||||
img := image.NewRGBA(image.Rect(0, 0, 300, 1000))
|
||||
buf = new(bytes.Buffer)
|
||||
err = jpeg.Encode(buf, img, nil)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
ext = ".jpg"
|
||||
}
|
||||
page := &manga.Page{
|
||||
Index: uint16(i),
|
||||
Contents: buf,
|
||||
Extension: ext,
|
||||
}
|
||||
pages = append(pages, page)
|
||||
}
|
||||
|
||||
// Expected: small pages to .webp, corrupted pages to .jpg (kept as is)
|
||||
expectedExtensions := []string{".webp", ".webp", ".jpg", ".webp", ".jpg"}
|
||||
// Even with split, corrupted pages can't be decoded so stay as is
|
||||
|
||||
return &manga.Chapter{
|
||||
FilePath: path,
|
||||
Pages: pages,
|
||||
}, expectedExtensions, nil
|
||||
// Heights: 2000, 4000, 6000, 8000, 10000, 12000, 14000, 16000, 18000, 20000
|
||||
// Without split, pages > webpMaxHeight (16383) are kept as .jpg
|
||||
// Pages 0-7 (2000-16000): should convert to .webp
|
||||
// Pages 8-9 (18000, 20000): > 16383, no split → kept as .jpg
|
||||
return chapter, []string{".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".webp", ".jpg", ".jpg"}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user