ci: Another attempt at reducing

Fix failing test

.github/workflows/test.yml

@@ -28,7 +28,17 @@ jobs:
           mv go-junit-report /usr/local/bin/
 
       - name: Run tests
-        run: go test -v 2>&1 ./... -coverprofile=coverage.txt | go-junit-report -set-exit-code > junit.xml
+        run: |
+          set -o pipefail
+          go test -v 2>&1 ./... -coverprofile=coverage.txt | tee test-results.txt
+      - name: Analyse test results
+        run: go-junit-report < test-results.txt > report.xml
+      - name: Upload test result artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: test-results
+          path: test-results.txt
+          retention-days: 7
       - name: Upload results to Codecov
         uses: codecov/codecov-action@v4
         with:

cbz/cbz_creator.go

@@ -30,10 +30,10 @@ func WriteChapterToCBZ(chapter *manga.Chapter, outputFilePath string) error {
 		var fileName string
 		if page.IsSplitted {
 			// Use the format page%03d-%02d for split pages
-			fileName = fmt.Sprintf("page_%04d-%02d%s", page.Index, page.SplitPartIndex, page.Extension)
+			fileName = fmt.Sprintf("%04d-%02d%s", page.Index, page.SplitPartIndex, page.Extension)
 		} else {
 			// Use the format page%03d for non-split pages
-			fileName = fmt.Sprintf("page_%04d%s", page.Index, page.Extension)
+			fileName = fmt.Sprintf("%04d%s", page.Index, page.Extension)
 		}
 
 		// Create a new file in the ZIP archive

cbz/cbz_creator_test.go

@@ -31,7 +31,7 @@ func TestWriteChapterToCBZ(t *testing.T) {
 				IsConverted:   true,
 				ConvertedTime: time.Now(),
 			},
-			expectedFiles: []string{"page_0000.jpg", "ComicInfo.xml", "Converted.txt"},
+			expectedFiles: []string{"0000.jpg", "ComicInfo.xml", "Converted.txt"},
 		},
 		//test case where there is only one page and no
 		{
@@ -45,7 +45,7 @@ func TestWriteChapterToCBZ(t *testing.T) {
 					},
 				},
 			},
-			expectedFiles: []string{"page_0000.jpg"},
+			expectedFiles: []string{"0000.jpg"},
 		},
 		{
 			name: "Multiple pages with ComicInfo",
@@ -64,7 +64,7 @@ func TestWriteChapterToCBZ(t *testing.T) {
 				},
 				ComicInfoXml: "<Series>Boundless Necromancer</Series>",
 			},
-			expectedFiles: []string{"page_0000.jpg", "page_0001.jpg", "ComicInfo.xml"},
+			expectedFiles: []string{"0000.jpg", "0001.jpg", "ComicInfo.xml"},
 		},
 		{
 			name: "Split page",
@@ -79,7 +79,7 @@ func TestWriteChapterToCBZ(t *testing.T) {
 					},
 				},
 			},
-			expectedFiles: []string{"page_0000-01.jpg"},
+			expectedFiles: []string{"0000-01.jpg"},
 		},
 	}
 

cmd/optimize_command.go

@@ -29,6 +29,7 @@ func init() {
 	command.Flags().Uint8P("quality", "q", 85, "Quality for conversion (0-100)")
 	command.Flags().IntP("parallelism", "n", 2, "Number of chapters to convert in parallel")
 	command.Flags().BoolP("override", "o", false, "Override the original CBZ files")
+	command.Flags().BoolP("split", "s", false, "Split long pages into smaller chunks")
 	command.PersistentFlags().VarP(
 		formatFlag,
 		"format", "f",
@@ -58,6 +59,11 @@ func ConvertCbzCommand(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("invalid quality value")
 	}
 
+	split, err := cmd.Flags().GetBool("split")
+	if err != nil {
+		return fmt.Errorf("invalid split value")
+	}
+
 	parallelism, err := cmd.Flags().GetInt("parallelism")
 	if err != nil || parallelism < 1 {
 		return fmt.Errorf("invalid parallelism value")
@@ -86,7 +92,13 @@ func ConvertCbzCommand(cmd *cobra.Command, args []string) error {
 		go func() {
 			defer wg.Done()
 			for path := range fileChan {
-				err := utils.Optimize(chapterConverter, path, quality, override)
+				err := utils.Optimize(&utils.OptimizeOptions{
+					ChapterConverter: chapterConverter,
+					Path:             path,
+					Quality:          quality,
+					Override:         override,
+					Split:            split,
+				})
 				if err != nil {
 					errorChan <- fmt.Errorf("error processing file %s: %w", path, err)
 				}

cmd/optimize_command_test.go

@@ -17,17 +17,16 @@ import (
 // MockConverter is a mock implementation of the Converter interface
 type MockConverter struct{}
 
-func (m *MockConverter) Format() constant.ConversionFormat {
-	return constant.WebP
-}
-
-func (m *MockConverter) ConvertChapter(chapter *manga.Chapter, quality uint8, progress func(string)) (*manga.Chapter, error) {
-	// Simulate conversion by setting the IsConverted flag
+func (m *MockConverter) ConvertChapter(chapter *manga.Chapter, quality uint8, split bool, progress func(message string, current uint32, total uint32)) (*manga.Chapter, error) {
 	chapter.IsConverted = true
 	chapter.ConvertedTime = time.Now()
 	return chapter, nil
 }
 
+func (m *MockConverter) Format() constant.ConversionFormat {
+	return constant.WebP
+}
+
 func (m *MockConverter) PrepareConverter() error {
 	return nil
 }
@@ -79,6 +78,7 @@ func TestConvertCbzCommand(t *testing.T) {
 	cmd.Flags().Uint8P("quality", "q", 85, "Quality for conversion (0-100)")
 	cmd.Flags().IntP("parallelism", "n", 2, "Number of chapters to convert in parallel")
 	cmd.Flags().BoolP("override", "o", false, "Override the original CBZ files")
+	cmd.Flags().BoolP("split", "s", false, "Split long pages into smaller chunks")
 
 	// Execute the command
 	err = ConvertCbzCommand(cmd, []string{tempDir})

cmd/watch_command.go

@@ -35,6 +35,9 @@ func init() {
 	command.Flags().BoolP("override", "o", true, "Override the original CBZ files")
 	_ = viper.BindPFlag("override", command.Flags().Lookup("override"))
 
+	command.Flags().BoolP("split", "s", false, "Split long pages into smaller chunks")
+	_ = viper.BindPFlag("split", command.Flags().Lookup("split"))
+
 	command.PersistentFlags().VarP(
 		formatFlag,
 		"format", "f",
@@ -61,6 +64,8 @@ func WatchCommand(_ *cobra.Command, args []string) error {
 
 	override := viper.GetBool("override")
 
+	split := viper.GetBool("split")
+
 	converterType := constant.FindConversionFormat(viper.GetString("format"))
 	chapterConverter, err := converter.Get(converterType)
 	if err != nil {
@@ -109,7 +114,13 @@ func WatchCommand(_ *cobra.Command, args []string) error {
 			for _, e := range event.Events {
 				switch e {
 				case inotifywaitgo.CLOSE_WRITE, inotifywaitgo.MOVE:
-					err := utils.Optimize(chapterConverter, event.Filename, quality, override)
+					err := utils.Optimize(&utils.OptimizeOptions{
+						ChapterConverter: chapterConverter,
+						Path:             path,
+						Quality:          quality,
+						Override:         override,
+						Split:            split,
+					})
 					if err != nil {
 						errors <- fmt.Errorf("error processing file %s: %w", event.Filename, err)
 					}

converter/converter.go

@@ -12,7 +12,10 @@ import (
 type Converter interface {
 	// Format of the converter
 	Format() (format constant.ConversionFormat)
-	ConvertChapter(chapter *manga.Chapter, quality uint8, progress func(string)) (*manga.Chapter, error)
+	// ConvertChapter converts a manga chapter to the specified format.
+	//
+	// Returns partial success where some pages are converted and some are not.
+	ConvertChapter(chapter *manga.Chapter, quality uint8, split bool, progress func(message string, current uint32, total uint32)) (*manga.Chapter, error)
 	PrepareConverter() error
 }
 

converter/converter_test.go

@@ -2,7 +2,9 @@ package converter
 
 import (
 	"bytes"
+	"github.com/belphemur/CBZOptimizer/converter/constant"
 	"github.com/belphemur/CBZOptimizer/manga"
+	"golang.org/x/exp/slices"
 	"image"
 	"image/jpeg"
 	"os"
@@ -12,20 +14,46 @@ import (
 func TestConvertChapter(t *testing.T) {
 
 	testCases := []struct {
-		name           string
-		genTestChapter func(path string) (*manga.Chapter, error)
+		name                 string
+		genTestChapter       func(path string) (*manga.Chapter, error)
+		split                bool
+		expectFailure        []constant.ConversionFormat
+		expectPartialSuccess []constant.ConversionFormat
 	}{
 		{
-			name:           "All split pages",
-			genTestChapter: genBigPages,
+			name:                 "All split pages",
+			genTestChapter:       genHugePage,
+			split:                true,
+			expectFailure:        []constant.ConversionFormat{},
+			expectPartialSuccess: []constant.ConversionFormat{},
 		},
 		{
-			name:           "No split pages",
-			genTestChapter: genSmallPages,
+			name:                 "Big Pages, no split",
+			genTestChapter:       genHugePage,
+			split:                false,
+			expectFailure:        []constant.ConversionFormat{constant.WebP},
+			expectPartialSuccess: []constant.ConversionFormat{},
 		},
 		{
-			name:           "Mix of split and no split pages",
-			genTestChapter: genMixSmallBig,
+			name:                 "No split pages",
+			genTestChapter:       genSmallPages,
+			split:                false,
+			expectFailure:        []constant.ConversionFormat{},
+			expectPartialSuccess: []constant.ConversionFormat{},
+		},
+		{
+			name:                 "Mix of split and no split pages",
+			genTestChapter:       genMixSmallBig,
+			split:                true,
+			expectFailure:        []constant.ConversionFormat{},
+			expectPartialSuccess: []constant.ConversionFormat{},
+		},
+		{
+			name:                 "Mix of Huge and small page",
+			genTestChapter:       genMixSmallHuge,
+			split:                false,
+			expectFailure:        []constant.ConversionFormat{},
+			expectPartialSuccess: []constant.ConversionFormat{constant.WebP},
 		},
 	}
 	// Load test genTestChapter from testdata
@@ -50,19 +78,33 @@ func TestConvertChapter(t *testing.T) {
 
 					quality := uint8(80)
 
-					progress := func(msg string) {
+					progress := func(msg string, current uint32, total uint32) {
 						t.Log(msg)
 					}
 
-					convertedChapter, err := converter.ConvertChapter(chapter, quality, progress)
+					convertedChapter, err := converter.ConvertChapter(chapter, quality, tc.split, progress)
 					if err != nil {
+						if convertedChapter != nil && slices.Contains(tc.expectPartialSuccess, converter.Format()) {
+							t.Logf("Partial success to convert genTestChapter: %v", err)
+							return
+						}
+						if slices.Contains(tc.expectFailure, converter.Format()) {
+							t.Logf("Expected failure to convert genTestChapter: %v", err)
+							return
+						}
 						t.Fatalf("failed to convert genTestChapter: %v", err)
+					} else if slices.Contains(tc.expectFailure, converter.Format()) {
+						t.Fatalf("expected failure to convert genTestChapter didn't happen")
 					}
 
 					if len(convertedChapter.Pages) == 0 {
 						t.Fatalf("no pages were converted")
 					}
 
+					if len(convertedChapter.Pages) != len(chapter.Pages) {
+						t.Fatalf("converted chapter has different number of pages")
+					}
+
 					for _, page := range convertedChapter.Pages {
 						if page.Extension != ".webp" {
 							t.Errorf("page %d was not converted to webp format", page.Index)
@@ -74,7 +116,7 @@ func TestConvertChapter(t *testing.T) {
 	}
 }
 
-func genBigPages(path string) (*manga.Chapter, error) {
+func genHugePage(path string) (*manga.Chapter, error) {
 	file, err := os.Open(path)
 	if err != nil {
 		return nil, err
@@ -82,8 +124,8 @@ func genBigPages(path string) (*manga.Chapter, error) {
 	defer file.Close()
 
 	var pages []*manga.Page
-	for i := 0; i < 5; i++ { // Assuming there are 5 pages for the test
-		img := image.NewRGBA(image.Rect(0, 0, 300, 10000))
+	for i := 0; i < 1; i++ { // Assuming there are 5 pages for the test
+		img := image.NewRGBA(image.Rect(0, 0, 1, 17000))
 		buf := new(bytes.Buffer)
 		err := jpeg.Encode(buf, img, nil)
 		if err != nil {
@@ -160,3 +202,32 @@ func genMixSmallBig(path string) (*manga.Chapter, error) {
 		Pages:    pages,
 	}, nil
 }
+
+func genMixSmallHuge(path string) (*manga.Chapter, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	var pages []*manga.Page
+	for i := 0; i < 10; i++ { // Assuming there are 5 pages for the test
+		img := image.NewRGBA(image.Rect(0, 0, 1, 2000*(i+1)))
+		buf := new(bytes.Buffer)
+		err := jpeg.Encode(buf, img, nil)
+		if err != nil {
+			return nil, err
+		}
+		page := &manga.Page{
+			Index:     uint16(i),
+			Contents:  buf,
+			Extension: ".jpg",
+		}
+		pages = append(pages, page)
+	}
+
+	return &manga.Chapter{
+		FilePath: path,
+		Pages:    pages,
+	}, nil
+}

converter/errors/converter_errors.go

@@ -0,0 +1,13 @@
+package errors
+
+type PageIgnoredError struct {
+	s string
+}
+
+func (e *PageIgnoredError) Error() string {
+	return e.s
+}
+
+func NewPageIgnored(text string) error {
+	return &PageIgnoredError{text}
+}

converter/webp/webp_converter.go

@@ -2,9 +2,11 @@ package webp
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"github.com/belphemur/CBZOptimizer/converter/constant"
-	packer2 "github.com/belphemur/CBZOptimizer/manga"
+	converterrors "github.com/belphemur/CBZOptimizer/converter/errors"
+	"github.com/belphemur/CBZOptimizer/manga"
 	"github.com/oliamb/cutter"
 	"golang.org/x/exp/slices"
 	_ "golang.org/x/image/webp"
@@ -17,6 +19,8 @@ import (
 	"sync/atomic"
 )
 
+const webpMaxHeight = 16383
+
 type Converter struct {
 	maxHeight  int
 	cropHeight int
@@ -48,7 +52,7 @@ func (converter *Converter) PrepareConverter() error {
 	return nil
 }
 
-func (converter *Converter) ConvertChapter(chapter *packer2.Chapter, quality uint8, progress func(string)) (*packer2.Chapter, error) {
+func (converter *Converter) ConvertChapter(chapter *manga.Chapter, quality uint8, split bool, progress func(message string, current uint32, total uint32)) (*manga.Chapter, error) {
 	err := converter.PrepareConverter()
 	if err != nil {
 		return nil, err
@@ -57,7 +61,7 @@ func (converter *Converter) ConvertChapter(chapter *packer2.Chapter, quality uin
 	var wgConvertedPages sync.WaitGroup
 	maxGoroutines := runtime.NumCPU()
 
-	pagesChan := make(chan *packer2.PageContainer, maxGoroutines)
+	pagesChan := make(chan *manga.PageContainer, maxGoroutines)
 	errChan := make(chan error, maxGoroutines)
 
 	var wgPages sync.WaitGroup
@@ -65,13 +69,13 @@ func (converter *Converter) ConvertChapter(chapter *packer2.Chapter, quality uin
 
 	guard := make(chan struct{}, maxGoroutines)
 	pagesMutex := sync.Mutex{}
-	var pages []*packer2.Page
+	var pages []*manga.Page
 	var totalPages = uint32(len(chapter.Pages))
 
 	go func() {
 		for page := range pagesChan {
 			guard <- struct{}{} // would block if guard channel is already filled
-			go func(pageToConvert *packer2.PageContainer) {
+			go func(pageToConvert *manga.PageContainer) {
 				defer wgConvertedPages.Done()
 				convertedPage, err := converter.convertPage(pageToConvert, quality)
 				if err != nil {
@@ -93,7 +97,7 @@ func (converter *Converter) ConvertChapter(chapter *packer2.Chapter, quality uin
 				}
 				pagesMutex.Lock()
 				pages = append(pages, convertedPage.Page)
-				progress(fmt.Sprintf("Converted %d/%d pages to %s format", len(pages), totalPages, converter.Format()))
+				progress(fmt.Sprintf("Converted %d/%d pages to %s format", len(pages), totalPages, converter.Format()), uint32(len(pages)), totalPages)
 				pagesMutex.Unlock()
 				<-guard
 			}(page)
@@ -101,31 +105,36 @@ func (converter *Converter) ConvertChapter(chapter *packer2.Chapter, quality uin
 	}()
 
 	for _, page := range chapter.Pages {
-		go func(page *packer2.Page) {
+		go func(page *manga.Page) {
 			defer wgPages.Done()
 
-			splitNeeded, img, format, err := converter.checkPageNeedsSplit(page)
+			splitNeeded, img, format, err := converter.checkPageNeedsSplit(page, split)
 			if err != nil {
-				errChan <- fmt.Errorf("error checking if page %d of genTestChapter %s needs split: %v", page.Index, chapter.FilePath, err)
+				errChan <- err
+				// Partial error in this case, we want the page, but not converting it
+				if img != nil {
+					wgConvertedPages.Add(1)
+					pagesChan <- manga.NewContainer(page, img, format, false)
+				}
 				return
 			}
 
 			if !splitNeeded {
 				wgConvertedPages.Add(1)
-				pagesChan <- packer2.NewContainer(page, img, format)
+				pagesChan <- manga.NewContainer(page, img, format, true)
 				return
 			}
 			images, err := converter.cropImage(img)
 			if err != nil {
-				errChan <- fmt.Errorf("error converting page %d of genTestChapter %s to webp: %v", page.Index, chapter.FilePath, err)
+				errChan <- err
 				return
 			}
 
 			atomic.AddUint32(&totalPages, uint32(len(images)-1))
 			for i, img := range images {
-				page := &packer2.Page{Index: page.Index, IsSplitted: true, SplitPartIndex: uint16(i)}
+				page := &manga.Page{Index: page.Index, IsSplitted: true, SplitPartIndex: uint16(i)}
 				wgConvertedPages.Add(1)
-				pagesChan <- packer2.NewContainer(page, img, "N/A")
+				pagesChan <- manga.NewContainer(page, img, "N/A", true)
 			}
 		}(page)
 	}
@@ -140,11 +149,12 @@ func (converter *Converter) ConvertChapter(chapter *packer2.Chapter, quality uin
 		errList = append(errList, err)
 	}
 
+	var aggregatedError error = nil
 	if len(errList) > 0 {
-		return nil, fmt.Errorf("encountered errors: %v", errList)
+		aggregatedError = errors.Join(errList...)
 	}
 
-	slices.SortFunc(pages, func(a, b *packer2.Page) int {
+	slices.SortFunc(pages, func(a, b *manga.Page) int {
 		if a.Index == b.Index {
 			return int(b.SplitPartIndex - a.SplitPartIndex)
 		}
@@ -154,7 +164,7 @@ func (converter *Converter) ConvertChapter(chapter *packer2.Chapter, quality uin
 
 	runtime.GC()
 
-	return chapter, nil
+	return chapter, aggregatedError
 }
 
 func (converter *Converter) cropImage(img image.Image) ([]image.Image, error) {
@@ -190,7 +200,7 @@ func (converter *Converter) cropImage(img image.Image) ([]image.Image, error) {
 	return parts, nil
 }
 
-func (converter *Converter) checkPageNeedsSplit(page *packer2.Page) (bool, image.Image, string, error) {
+func (converter *Converter) checkPageNeedsSplit(page *manga.Page, splitRequested bool) (bool, image.Image, string, error) {
 	reader := io.Reader(bytes.NewBuffer(page.Contents.Bytes()))
 	img, format, err := image.Decode(reader)
 	if err != nil {
@@ -200,13 +210,19 @@ func (converter *Converter) checkPageNeedsSplit(page *packer2.Page) (bool, image
 	bounds := img.Bounds()
 	height := bounds.Dy()
 
-	return height >= converter.maxHeight, img, format, nil
+	if height >= webpMaxHeight && !splitRequested {
+		return false, img, format, converterrors.NewPageIgnored(fmt.Sprintf("page %d is too tall [max: %dpx] to be converted to webp format", page.Index, webpMaxHeight))
+	}
+	return height >= converter.maxHeight && splitRequested, img, format, nil
 }
 
-func (converter *Converter) convertPage(container *packer2.PageContainer, quality uint8) (*packer2.PageContainer, error) {
+func (converter *Converter) convertPage(container *manga.PageContainer, quality uint8) (*manga.PageContainer, error) {
 	if container.Format == "webp" {
 		return container, nil
 	}
+	if !container.IsToBeConverted {
+		return container, nil
+	}
 	converted, err := converter.convert(container.Image, uint(quality))
 	if err != nil {
 		return nil, err

manga/page_container.go

@@ -10,8 +10,10 @@ type PageContainer struct {
 	Image image.Image
 	// Format is a string representing the format of the image (e.g., "png", "jpeg", "webp").
 	Format string
+	// IsToBeConverted is a boolean flag indicating whether the image needs to be converted to another format.
+	IsToBeConverted bool
 }
 
-func NewContainer(Page *Page, img image.Image, format string) *PageContainer {
-	return &PageContainer{Page: Page, Image: img, Format: format}
+func NewContainer(Page *Page, img image.Image, format string, isToBeConverted bool) *PageContainer {
+	return &PageContainer{Page: Page, Image: img, Format: format, IsToBeConverted: isToBeConverted}
 }

utils/optimize.go

@@ -1,41 +1,60 @@
 package utils
 
 import (
+	"errors"
 	"fmt"
 	"github.com/belphemur/CBZOptimizer/cbz"
 	"github.com/belphemur/CBZOptimizer/converter"
+	errors2 "github.com/belphemur/CBZOptimizer/converter/errors"
 	"log"
 	"strings"
 )
 
+type OptimizeOptions struct {
+	ChapterConverter converter.Converter
+	Path             string
+	Quality          uint8
+	Override         bool
+	Split            bool
+}
+
 // Optimize optimizes a CBZ file using the specified converter.
-func Optimize(chapterConverter converter.Converter, path string, quality uint8, override bool) error {
-	log.Printf("Processing file: %s\n", path)
+func Optimize(options *OptimizeOptions) error {
+	log.Printf("Processing file: %s\n", options.Path)
 
 	// Load the chapter
-	chapter, err := cbz.LoadChapter(path)
+	chapter, err := cbz.LoadChapter(options.Path)
 	if err != nil {
 		return fmt.Errorf("failed to load chapter: %v", err)
 	}
 
 	if chapter.IsConverted {
-		log.Printf("Chapter already converted: %s", path)
+		log.Printf("Chapter already converted: %s", options.Path)
 		return nil
 	}
 
 	// Convert the chapter
-	convertedChapter, err := chapterConverter.ConvertChapter(chapter, quality, func(msg string) {
-		log.Printf("[%s]%s", path, msg)
+	convertedChapter, err := options.ChapterConverter.ConvertChapter(chapter, options.Quality, options.Split, func(msg string, current uint32, total uint32) {
+		if current%10 == 0 || current == total {
+			log.Printf("[%s] Converting: %d/%d", chapter.FilePath, current, total)
+		}
 	})
 	if err != nil {
-		return fmt.Errorf("failed to convert chapter: %v", err)
+		var pageIgnoredError *errors2.PageIgnoredError
+		if !errors.As(err, &pageIgnoredError) {
+			return fmt.Errorf("failed to convert chapter: %v", err)
+		}
 	}
+	if convertedChapter == nil {
+		return fmt.Errorf("failed to convert chapter")
+	}
+
 	convertedChapter.SetConverted()
 
 	// Write the converted chapter back to a CBZ file
-	outputPath := path
-	if !override {
-		outputPath = strings.TrimSuffix(path, ".cbz") + "_converted.cbz"
+	outputPath := options.Path
+	if !options.Override {
+		outputPath = strings.TrimSuffix(options.Path, ".cbz") + "_converted.cbz"
 	}
 	err = cbz.WriteChapterToCBZ(convertedChapter, outputPath)
 	if err != nil {