mirror of
https://github.com/Belphemur/CBZOptimizer.git
synced 2025-10-14 04:28:51 +02:00
fix(memory): fix possible memory leak and add better tests
This commit is contained in:
@@ -41,7 +41,7 @@ func TestConvertCbzCommand(t *testing.T) {
|
|||||||
defer errs.CaptureGeneric(&err, os.RemoveAll, tempDir, "failed to remove temporary directory")
|
defer errs.CaptureGeneric(&err, os.RemoveAll, tempDir, "failed to remove temporary directory")
|
||||||
|
|
||||||
// Locate the testdata directory
|
// Locate the testdata directory
|
||||||
testdataDir := filepath.Join("../testdata")
|
testdataDir := filepath.Join("../../../testdata")
|
||||||
if _, err := os.Stat(testdataDir); os.IsNotExist(err) {
|
if _, err := os.Stat(testdataDir); os.IsNotExist(err) {
|
||||||
t.Fatalf("testdata directory not found")
|
t.Fatalf("testdata directory not found")
|
||||||
}
|
}
|
||||||
|
3
go.mod
3
go.mod
@@ -12,6 +12,7 @@ require (
|
|||||||
github.com/samber/lo v1.49.1
|
github.com/samber/lo v1.49.1
|
||||||
github.com/spf13/cobra v1.8.1
|
github.com/spf13/cobra v1.8.1
|
||||||
github.com/spf13/viper v1.19.0
|
github.com/spf13/viper v1.19.0
|
||||||
|
github.com/stretchr/testify v1.10.0
|
||||||
github.com/thediveo/enumflag/v2 v2.0.7
|
github.com/thediveo/enumflag/v2 v2.0.7
|
||||||
golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac
|
golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac
|
||||||
golang.org/x/image v0.24.0
|
golang.org/x/image v0.24.0
|
||||||
@@ -20,6 +21,7 @@ require (
|
|||||||
require (
|
require (
|
||||||
github.com/andybalholm/brotli v1.1.0 // indirect
|
github.com/andybalholm/brotli v1.1.0 // indirect
|
||||||
github.com/belphemur/go-binwrapper v0.0.0-20240827152605-33977349b1f0 // indirect
|
github.com/belphemur/go-binwrapper v0.0.0-20240827152605-33977349b1f0 // indirect
|
||||||
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||||
github.com/dsnet/compress v0.0.1 // indirect
|
github.com/dsnet/compress v0.0.1 // indirect
|
||||||
github.com/fsnotify/fsnotify v1.7.0 // indirect
|
github.com/fsnotify/fsnotify v1.7.0 // indirect
|
||||||
github.com/golang/snappy v0.0.4 // indirect
|
github.com/golang/snappy v0.0.4 // indirect
|
||||||
@@ -33,6 +35,7 @@ require (
|
|||||||
github.com/nwaples/rardecode v1.1.3 // indirect
|
github.com/nwaples/rardecode v1.1.3 // indirect
|
||||||
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
|
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
|
||||||
github.com/pierrec/lz4/v4 v4.1.21 // indirect
|
github.com/pierrec/lz4/v4 v4.1.21 // indirect
|
||||||
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||||
github.com/sagikazarmark/locafero v0.4.0 // indirect
|
github.com/sagikazarmark/locafero v0.4.0 // indirect
|
||||||
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
|
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
|
||||||
github.com/sourcegraph/conc v0.3.0 // indirect
|
github.com/sourcegraph/conc v0.3.0 // indirect
|
||||||
|
@@ -17,3 +17,11 @@ type PageContainer struct {
|
|||||||
func NewContainer(Page *Page, img image.Image, format string, isToBeConverted bool) *PageContainer {
|
func NewContainer(Page *Page, img image.Image, format string, isToBeConverted bool) *PageContainer {
|
||||||
return &PageContainer{Page: Page, Image: img, Format: format, IsToBeConverted: isToBeConverted}
|
return &PageContainer{Page: Page, Image: img, Format: format, IsToBeConverted: isToBeConverted}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Close releases resources held by the PageContainer
|
||||||
|
func (pc *PageContainer) Close() {
|
||||||
|
pc.Image = nil
|
||||||
|
if pc.Page != nil && pc.Page.Contents != nil {
|
||||||
|
pc.Page.Contents.Reset()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@@ -4,19 +4,19 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"image"
|
||||||
|
_ "image/jpeg"
|
||||||
|
"image/png"
|
||||||
|
"runtime"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
|
||||||
"github.com/belphemur/CBZOptimizer/v2/internal/manga"
|
"github.com/belphemur/CBZOptimizer/v2/internal/manga"
|
||||||
"github.com/belphemur/CBZOptimizer/v2/pkg/converter/constant"
|
"github.com/belphemur/CBZOptimizer/v2/pkg/converter/constant"
|
||||||
converterrors "github.com/belphemur/CBZOptimizer/v2/pkg/converter/errors"
|
converterrors "github.com/belphemur/CBZOptimizer/v2/pkg/converter/errors"
|
||||||
"github.com/oliamb/cutter"
|
"github.com/oliamb/cutter"
|
||||||
"golang.org/x/exp/slices"
|
"golang.org/x/exp/slices"
|
||||||
_ "golang.org/x/image/webp"
|
_ "golang.org/x/image/webp"
|
||||||
"image"
|
|
||||||
_ "image/jpeg"
|
|
||||||
"image/png"
|
|
||||||
"io"
|
|
||||||
"runtime"
|
|
||||||
"sync"
|
|
||||||
"sync/atomic"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const webpMaxHeight = 16383
|
const webpMaxHeight = 16383
|
||||||
@@ -63,6 +63,7 @@ func (converter *Converter) ConvertChapter(chapter *manga.Chapter, quality uint8
|
|||||||
|
|
||||||
pagesChan := make(chan *manga.PageContainer, maxGoroutines)
|
pagesChan := make(chan *manga.PageContainer, maxGoroutines)
|
||||||
errChan := make(chan error, maxGoroutines)
|
errChan := make(chan error, maxGoroutines)
|
||||||
|
doneChan := make(chan struct{})
|
||||||
|
|
||||||
var wgPages sync.WaitGroup
|
var wgPages sync.WaitGroup
|
||||||
wgPages.Add(len(chapter.Pages))
|
wgPages.Add(len(chapter.Pages))
|
||||||
@@ -72,23 +73,27 @@ func (converter *Converter) ConvertChapter(chapter *manga.Chapter, quality uint8
|
|||||||
var pages []*manga.Page
|
var pages []*manga.Page
|
||||||
var totalPages = uint32(len(chapter.Pages))
|
var totalPages = uint32(len(chapter.Pages))
|
||||||
|
|
||||||
|
// Start the worker pool
|
||||||
go func() {
|
go func() {
|
||||||
for page := range pagesChan {
|
for page := range pagesChan {
|
||||||
guard <- struct{}{} // would block if guard channel is already filled
|
guard <- struct{}{} // would block if guard channel is already filled
|
||||||
go func(pageToConvert *manga.PageContainer) {
|
go func(pageToConvert *manga.PageContainer) {
|
||||||
defer wgConvertedPages.Done()
|
defer func() {
|
||||||
|
wgConvertedPages.Done()
|
||||||
|
pageToConvert.Close() // Clean up resources
|
||||||
|
<-guard
|
||||||
|
}()
|
||||||
|
|
||||||
convertedPage, err := converter.convertPage(pageToConvert, quality)
|
convertedPage, err := converter.convertPage(pageToConvert, quality)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if convertedPage == nil {
|
if convertedPage == nil {
|
||||||
errChan <- err
|
errChan <- err
|
||||||
<-guard
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
buffer := new(bytes.Buffer)
|
buffer := new(bytes.Buffer)
|
||||||
err := png.Encode(buffer, convertedPage.Image)
|
err := png.Encode(buffer, convertedPage.Image)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errChan <- err
|
errChan <- err
|
||||||
<-guard
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
convertedPage.Page.Contents = buffer
|
convertedPage.Page.Contents = buffer
|
||||||
@@ -99,11 +104,12 @@ func (converter *Converter) ConvertChapter(chapter *manga.Chapter, quality uint8
|
|||||||
pages = append(pages, convertedPage.Page)
|
pages = append(pages, convertedPage.Page)
|
||||||
progress(fmt.Sprintf("Converted %d/%d pages to %s format", len(pages), totalPages, converter.Format()), uint32(len(pages)), totalPages)
|
progress(fmt.Sprintf("Converted %d/%d pages to %s format", len(pages), totalPages, converter.Format()), uint32(len(pages)), totalPages)
|
||||||
pagesMutex.Unlock()
|
pagesMutex.Unlock()
|
||||||
<-guard
|
|
||||||
}(page)
|
}(page)
|
||||||
}
|
}
|
||||||
|
close(doneChan)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
// Process pages
|
||||||
for _, page := range chapter.Pages {
|
for _, page := range chapter.Pages {
|
||||||
go func(page *manga.Page) {
|
go func(page *manga.Page) {
|
||||||
defer wgPages.Done()
|
defer wgPages.Done()
|
||||||
@@ -111,7 +117,6 @@ func (converter *Converter) ConvertChapter(chapter *manga.Chapter, quality uint8
|
|||||||
splitNeeded, img, format, err := converter.checkPageNeedsSplit(page, split)
|
splitNeeded, img, format, err := converter.checkPageNeedsSplit(page, split)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errChan <- err
|
errChan <- err
|
||||||
// Partial error in this case, we want the page, but not converting it
|
|
||||||
if img != nil {
|
if img != nil {
|
||||||
wgConvertedPages.Add(1)
|
wgConvertedPages.Add(1)
|
||||||
pagesChan <- manga.NewContainer(page, img, format, false)
|
pagesChan <- manga.NewContainer(page, img, format, false)
|
||||||
@@ -124,6 +129,7 @@ func (converter *Converter) ConvertChapter(chapter *manga.Chapter, quality uint8
|
|||||||
pagesChan <- manga.NewContainer(page, img, format, true)
|
pagesChan <- manga.NewContainer(page, img, format, true)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
images, err := converter.cropImage(img)
|
images, err := converter.cropImage(img)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errChan <- err
|
errChan <- err
|
||||||
@@ -132,17 +138,25 @@ func (converter *Converter) ConvertChapter(chapter *manga.Chapter, quality uint8
|
|||||||
|
|
||||||
atomic.AddUint32(&totalPages, uint32(len(images)-1))
|
atomic.AddUint32(&totalPages, uint32(len(images)-1))
|
||||||
for i, img := range images {
|
for i, img := range images {
|
||||||
page := &manga.Page{Index: page.Index, IsSplitted: true, SplitPartIndex: uint16(i)}
|
newPage := &manga.Page{
|
||||||
|
Index: page.Index,
|
||||||
|
IsSplitted: true,
|
||||||
|
SplitPartIndex: uint16(i),
|
||||||
|
}
|
||||||
wgConvertedPages.Add(1)
|
wgConvertedPages.Add(1)
|
||||||
pagesChan <- manga.NewContainer(page, img, "N/A", true)
|
pagesChan <- manga.NewContainer(newPage, img, "N/A", true)
|
||||||
}
|
}
|
||||||
}(page)
|
}(page)
|
||||||
}
|
}
|
||||||
|
|
||||||
wgPages.Wait()
|
wgPages.Wait()
|
||||||
wgConvertedPages.Wait()
|
|
||||||
close(pagesChan)
|
close(pagesChan)
|
||||||
|
|
||||||
|
// Wait for all conversions to complete
|
||||||
|
<-doneChan
|
||||||
|
wgConvertedPages.Wait()
|
||||||
close(errChan)
|
close(errChan)
|
||||||
|
close(guard)
|
||||||
|
|
||||||
var errList []error
|
var errList []error
|
||||||
for err := range errChan {
|
for err := range errChan {
|
||||||
@@ -156,9 +170,9 @@ func (converter *Converter) ConvertChapter(chapter *manga.Chapter, quality uint8
|
|||||||
|
|
||||||
slices.SortFunc(pages, func(a, b *manga.Page) int {
|
slices.SortFunc(pages, func(a, b *manga.Page) int {
|
||||||
if a.Index == b.Index {
|
if a.Index == b.Index {
|
||||||
return int(b.SplitPartIndex - a.SplitPartIndex)
|
return int(a.SplitPartIndex) - int(b.SplitPartIndex)
|
||||||
}
|
}
|
||||||
return int(b.Index - a.Index)
|
return int(a.Index) - int(b.Index)
|
||||||
})
|
})
|
||||||
chapter.Pages = pages
|
chapter.Pages = pages
|
||||||
|
|
||||||
@@ -201,7 +215,7 @@ func (converter *Converter) cropImage(img image.Image) ([]image.Image, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (converter *Converter) checkPageNeedsSplit(page *manga.Page, splitRequested bool) (bool, image.Image, string, error) {
|
func (converter *Converter) checkPageNeedsSplit(page *manga.Page, splitRequested bool) (bool, image.Image, string, error) {
|
||||||
reader := io.Reader(bytes.NewBuffer(page.Contents.Bytes()))
|
reader := bytes.NewBuffer(page.Contents.Bytes())
|
||||||
img, format, err := image.Decode(reader)
|
img, format, err := image.Decode(reader)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, nil, format, err
|
return false, nil, format, err
|
||||||
@@ -217,7 +231,9 @@ func (converter *Converter) checkPageNeedsSplit(page *manga.Page, splitRequested
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (converter *Converter) convertPage(container *manga.PageContainer, quality uint8) (*manga.PageContainer, error) {
|
func (converter *Converter) convertPage(container *manga.PageContainer, quality uint8) (*manga.PageContainer, error) {
|
||||||
if container.Format == "webp" {
|
// Fix WebP format detection (case insensitive)
|
||||||
|
if container.Format == "webp" || container.Format == "WEBP" {
|
||||||
|
container.Page.Extension = ".webp"
|
||||||
return container, nil
|
return container, nil
|
||||||
}
|
}
|
||||||
if !container.IsToBeConverted {
|
if !container.IsToBeConverted {
|
||||||
|
Reference in New Issue
Block a user