From b73c859b94a26ea27a9425feaf9cb987b6543261 Mon Sep 17 00:00:00 2001 From: Marco Giuntoni Date: Fri, 1 Mar 2024 16:05:06 +0100 Subject: [PATCH] code sources --- Compressor.cs | 435 +++++++++++++++++++++++++++++++++++ FileParser.cs | 46 ++++ ImageExtractor.cs | 205 +++++++++++++++++ ImageExtractor7.cs | 89 ++++++++ Logger.cs | 93 ++++++++ Program.cs | 552 +++++++++++++++++++++++++++++++++++++++++++++ README.md | 27 ++- 7 files changed, 1444 insertions(+), 3 deletions(-) create mode 100644 Compressor.cs create mode 100644 FileParser.cs create mode 100644 ImageExtractor.cs create mode 100644 ImageExtractor7.cs create mode 100644 Logger.cs create mode 100644 Program.cs diff --git a/Compressor.cs b/Compressor.cs new file mode 100644 index 0000000..a32bf9b --- /dev/null +++ b/Compressor.cs @@ -0,0 +1,435 @@ +using System; +using System.Linq; +using System.Drawing; +using Imazen.WebP; +using SharpCompress.Archives; +using SharpCompress.Archives.Rar; +using SharpCompress.Archives.Zip; +using System.IO; +using SharpCompress.Common; +using System.Threading.Tasks; +using System.Collections.Generic; +using System.Diagnostics; +using javax.swing.plaf; +using SharpCompress.Readers; +using SharpCompress.Writers; +using SharpCompress.Compressors.Xz; + +namespace ComicCompressor +{ + public class Compressor + { + private long last_mem = 0; + private Logger Logger { get; set; } + + public int Quality { get; set; } = 75; + + public Compressor(Logger logger) + { + Logger = logger; + } + + public void Compress(string filename, string outputFile = null) + { + Stopwatch sw = Stopwatch.StartNew(); + outputFile = Path.ChangeExtension(outputFile ?? filename, "cbz"); + + Process proc = Process.GetCurrentProcess(); + Logger.LogDebug($"M {proc.PrivateMemorySize64}", LogLevel.Verbose, col: Color.Green); + last_mem = proc.PrivateMemorySize64; + + //Logger.Log("Processing: " + filename, LogLevel.Verbose); + + IArchive archive = null; + //SharpCompress.Readers.ReaderOptions opt = new ReaderOptions() + //{ + + //} + + ArchiveType ArchiveType = ArchiveType.Zip; + using (Stream stream = File.OpenRead(filename)) + { + using (var reader = ReaderFactory.Open(stream)) + ArchiveType = reader.ArchiveType; + + } + + bool isSolid = false; + //if (filename.EndsWith(".cbr")) + if (ArchiveType == ArchiveType.Rar) + { + archive = RarArchive.Open(filename); + //foreach (var entry in archive.Entries) + //{ + // isSolid = isSolid || entry.IsSolid; + //} + isSolid = archive.IsSolid; + } + //else if (filename.EndsWith(".cbz")) + else if (ArchiveType == ArchiveType.Zip) + { + archive = ZipArchive.Open(filename); + } + + //Logger.Log($"Processing[{archive.Entries.Count()}]: " + filename, LogLevel.Verbose); + Logger.Log($"Processing[{archive.Entries.Count()}]: ", LogLevel.Verbose,false); + Logger.Log( filename, LogLevel.Verbose,col:Color.Green); + + double save = 0; + int num_pages =0; + if (isSolid) + { + using (Stream stream = File.OpenRead(filename)) + { + using (var reader = ReaderFactory.Open(stream)) + { + (save, num_pages) = ProcessReader(archive,reader, outputFile); + } + } + } + else + (save, num_pages) = ProcessArchive(archive, outputFile); + if (num_pages == 0) num_pages = 1; + long elapsed = sw.ElapsedMilliseconds; + Logger.Log($"\r\nFinished({save.ToString("0.##")}%) [{elapsed}][{elapsed/num_pages}]: " + filename, LogLevel.Verbose, col: Color.Green); + } + + private (double, int) ProcessReader(IArchive archive, IReader reader, string outputPath) + { + var encoder = new SimpleEncoder(); + + + bool cover = true; + double save = 0.0; + int nop = 0; + using (var output = ZipArchive.Create()) + { + var imageStreams = new List(); + + while (reader.MoveToNextEntry()) + { + if (!reader.Entry.IsDirectory) + { + //Console.WriteLine(reader.Entry.Key); + + if (reader.Entry.Key.StartsWith("__MACOSX")) + { + continue; + } + + using (var entryStream = reader.OpenEntryStream()) + { + + string entry_key_lower = reader.Entry.Key.ToLower(); + if (!entry_key_lower.EndsWith(".jpg") && + !entry_key_lower.EndsWith(".png") && + !entry_key_lower.EndsWith(".bmp") && + !entry_key_lower.EndsWith(".gif") && + !entry_key_lower.EndsWith(".jpeg")) + { + var ms = new MemoryStream(); + imageStreams.Add(ms); + entryStream.CopyTo(ms); + output.AddEntry(reader.Entry.Key, ms); + Logger.Log("o", LogLevel.Verbose, false, col: Color.PaleGreen); + continue; + } + + nop++; + + if (cover) + { + var ms = new MemoryStream(); + imageStreams.Add(ms); + entryStream.CopyTo(ms); + output.AddEntry(reader.Entry.Key, ms); + Logger.Log("C", LogLevel.Verbose, false, col: Color.Green); + cover = false; + continue; + } + + ProcessReaderEntry(reader, entryStream, imageStreams, output, encoder); + + } + } + } + + reader.Dispose(); + + Directory.CreateDirectory(Path.GetDirectoryName(outputPath)); + + output.SaveTo(outputPath, CompressionType.Deflate); + + //output.SaveTo(outputPath, CompressionType.None); + + imageStreams.ForEach(i => i.Dispose()); + } + + long output_length = new System.IO.FileInfo(outputPath).Length; + + save = 100 * (double)output_length / (double)archive.TotalSize; + + return (save,nop); + } + public int GetPlace(int value, int place) + { + return ((value % (place * 10)) - (value % place)) / place; + } + private (double, int) ProcessArchive(IArchive archive, string outputPath) + { + //var fileEntries = archive.Entries.Where(e => !e.IsDirectory); + var fileEntries = archive.Entries.Where(e => !e.IsDirectory).OrderBy(e => e.Key); + + var encoder = new SimpleEncoder(); + + bool cover = true; + double save = 0.0; + int nop = 0; + using (var output = ZipArchive.Create()) + { + var imageStreams = new List(); + int count = 0; + foreach (var entry in fileEntries) + { + (bool res, bool skipped) = ProcessEntry(entry, imageStreams, output, encoder, cover); + if (res) + cover = false; + count++; + if (!skipped) + nop++; + if (count % 10 == 0) + { + string digit = GetPlace(count, 10).ToString(); + Logger.Log(digit, LogLevel.Verbose, false, col: Color.Green); + } + + } + + archive.Dispose(); + + Directory.CreateDirectory(Path.GetDirectoryName(outputPath)); + + output.SaveTo(outputPath, CompressionType.Deflate); + + //output.SaveTo(outputPath, CompressionType.None); + + imageStreams.ForEach(i => i.Dispose()); + + output.Dispose(); + } + + long output_length = new System.IO.FileInfo(outputPath).Length; + + save = 100 * (double)output_length / (double)archive.TotalSize; + + return (save, nop); + } + + + private (bool, bool) ProcessEntry(IArchiveEntry entry, IList imageStreams, ZipArchive output, SimpleEncoder encoder, bool cover) + { + try + { + if (entry.IsEncrypted) + { + Logger.Log("X", LogLevel.Verbose, false, col: Color.Yellow); + return (false,true); + } + + + using (var stream = entry.OpenEntryStream()) + { + + if (entry.Key.StartsWith("__MACOSX")) + { + return (false,true); + } + + string entry_key_lower = entry.Key.ToLower(); + if (!entry_key_lower.EndsWith(".jpg") && + !entry_key_lower.EndsWith(".png") && + !entry_key_lower.EndsWith(".bmp") && + !entry_key_lower.EndsWith(".gif") && + !entry_key_lower.EndsWith(".jpeg")) + { + var ms = new MemoryStream(); + imageStreams.Add(ms); + stream.CopyTo(ms); + output.AddEntry(entry.Key, ms); + Logger.Log("o", LogLevel.Verbose, false, col: Color.Green); + return (false, true); + } + + if (cover) + { + var ms = new MemoryStream(); + imageStreams.Add(ms); + stream.CopyTo(ms); + output.AddEntry(entry.Key, ms); + Logger.Log("C", LogLevel.Verbose, false,col:Color.Green); + return (true, false); + } + + Bitmap bits; + + var raw_ms = new MemoryStream(); + stream.CopyTo(raw_ms); + try + { + bits = new Bitmap(raw_ms); + } + catch (Exception e) + { + Logger.LogError("Error parsing bitmap: " + entry.Key, col: Color.Red); + Logger.LogDebug(e, LogLevel.Error, col:Color.Red); + return (false, true); + } + + if (bits.PixelFormat != System.Drawing.Imaging.PixelFormat.Format24bppRgb) + { + var newBits = ChangePixelFormat(bits); + bits.Dispose(); + bits = newBits; + } + + try + { + var ms = new MemoryStream(); + encoder.Encode(bits, ms, Quality); + if (ms.Length < entry.Size) + { + imageStreams.Add(ms); + output.AddEntry(entry.Key + ".webp", ms); + Logger.Log(".", LogLevel.Verbose, false, col: Color.PaleGreen); + + // release not used stream + raw_ms.Dispose(); + + } + else + { + imageStreams.Add(raw_ms); + + raw_ms.Seek(0, SeekOrigin.Begin); + //stream.Seek(0,SeekOrigin.Begin); + output.AddEntry(entry.Key, raw_ms); + Logger.Log("=", LogLevel.Verbose, false, col: Color.Green); + + // release not used stream + ms.Dispose(); + } + } + catch (Exception e) + { + Logger.LogError("Error encoding entry: " + entry.Key, col: Color.Red); + Logger.LogDebug(e, LogLevel.Error, col: Color.Red); + } + finally + { + bits.Dispose(); + } + } + } + catch (Exception e) + { + Logger.LogError("Error Archive Extraction: " + entry.Key, col: Color.Red); + Logger.LogDebug(e, LogLevel.Error, col: Color.Red); + return (false, true); + } + + Process proc = Process.GetCurrentProcess(); + //Logger.LogDebug($"M {proc.PrivateMemorySize64} - D {proc.PrivateMemorySize64 - last_mem}", LogLevel.Verbose); + //last_mem = proc.PrivateMemorySize64; + return (false,false); + } + + private void ProcessReaderEntry(IReader reader,EntryStream entry, IList imageStreams, ZipArchive output, SimpleEncoder encoder) + { + try + { + var stream = entry; +// using (var stream = entry.OpenEntryStream()) + { + + + Bitmap bits; + + var raw_ms = new MemoryStream(); + imageStreams.Add(raw_ms); + stream.CopyTo(raw_ms); + try + { + bits = new Bitmap(raw_ms); + } + catch (Exception e) + { + Logger.LogError("Error parsing bitmap: " + reader.Entry.Key, col: Color.Red); + Logger.LogDebug(e, LogLevel.Error, col: Color.Red); + return ; + } + + if (bits.PixelFormat != System.Drawing.Imaging.PixelFormat.Format24bppRgb) + { + var newBits = ChangePixelFormat(bits); + bits.Dispose(); + bits = newBits; + } + + try + { + var ms = new MemoryStream(); + encoder.Encode(bits, ms, Quality); + if (ms.Length < reader.Entry.Size) + { + imageStreams.Add(ms); + output.AddEntry(reader.Entry.Key + ".webp", ms); + Logger.Log("s", LogLevel.Verbose, false, col: Color.Green); + } + else + { + raw_ms.Seek(0, SeekOrigin.Begin); + //stream.Seek(0,SeekOrigin.Begin); + output.AddEntry(reader.Entry.Key, raw_ms); + Logger.Log("=", LogLevel.Verbose, false, col: Color.Green); + } + } + catch (Exception e) + { + Logger.LogError("Error encoding entry: " + reader.Entry.Key, col: Color.Red); + Logger.LogDebug(e, LogLevel.Error, col: Color.Red); + } + finally + { + bits.Dispose(); + } + } + } + catch (Exception e) + { + Logger.LogError("Error Archive Extraction: " + reader.Entry.Key, col: Color.Red); + Logger.LogDebug(e, LogLevel.Error, col: Color.Red); + return; + } + + return ; + } + + private Bitmap ChangePixelFormat(Bitmap orig, System.Drawing.Imaging.PixelFormat format = System.Drawing.Imaging.PixelFormat.Format24bppRgb) + { + Bitmap clone = new Bitmap(orig.Width, orig.Height, format); + + using (Graphics gr = Graphics.FromImage(clone)) + { + gr.DrawImage(orig, new Rectangle(0, 0, clone.Width, clone.Height)); + } + + return clone; + + } + + public void CompressPdf(string filename, string outputFile = null) + { + new ImageExtractor(Logger).ExtractImagesFromFile(filename, outputFile); + } + } +} \ No newline at end of file diff --git a/FileParser.cs b/FileParser.cs new file mode 100644 index 0000000..2dec604 --- /dev/null +++ b/FileParser.cs @@ -0,0 +1,46 @@ +using System.Collections.Generic; +using System.IO; + +namespace ComicCompressor +{ + public class FileParser + { + public IList ListAllFiles(string input, bool recursive) + { + Queue unexplored; + IList files = new List(); + + if (Directory.Exists(input)) + { + unexplored = new Queue(Directory.GetFileSystemEntries(input)); + } + else + { + // No directory but must exist so it is a file + unexplored = new Queue(); + unexplored.Enqueue(input); + } + + while(unexplored.Count > 0) + { + var file = unexplored.Dequeue(); + + if (File.Exists(file)) + { + files.Add(file); + } + else if (recursive) + { + foreach (var f in Directory.GetFileSystemEntries(file)) + { + unexplored.Enqueue(f); + } + } + } + + return files; + + } + } + +} \ No newline at end of file diff --git a/ImageExtractor.cs b/ImageExtractor.cs new file mode 100644 index 0000000..8ce5c8e --- /dev/null +++ b/ImageExtractor.cs @@ -0,0 +1,205 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Drawing; +using System.IO; +using System.Runtime.ConstrainedExecution; +using ikvm.extensions; +using Imazen.WebP; + +using iTextSharp.text.pdf; +using iTextSharp.text.pdf.parser; + +using SharpCompress.Archives; +using SharpCompress.Archives.Zip; +using SharpCompress.Common; + +using Path = System.IO.Path; + +namespace ComicCompressor +{ + /// + /// Helper lass to dump all images from a PDF into separate files + /// + public class ImageExtractor : IRenderListener + { + int _currentPage = 1; + int _imageCount = 0; + readonly string _outputFilePrefix = ""; + //readonly string _outputFolder; + //readonly bool _overwriteExistingFiles; + private bool Cover = true; + private Logger Logger { get; set; } + SimpleEncoder encoder = new SimpleEncoder(); + public int Quality { get; set; } = 75; + private List imageStreams = null; + public ZipArchive output = null; + + public ImageExtractor(Logger logger) + { + Logger = logger; + //_outputFilePrefix = outputFilePrefix; + //_outputFolder = outputFolder; + //_overwriteExistingFiles = overwriteExistingFiles; + } + public int GetPlace(int value, int place) + { + return ((value % (place * 10)) - (value % place)) / place; + } + /// + /// Extract all images from a PDF file + /// + /// Full path and file name of PDF file + /// Basic name of exported files. If null then uses same name as PDF file. + /// Where to save images. If null or empty then uses same folder as PDF file. + /// True to overwrite existing image files, false to skip past them + /// Count of number of images extracted. + public int ExtractImagesFromFile(string pdfPath, string outputFolder, int quality = 75) + { + // Handle setting of any default values + //outputFilePrefix = outputFilePrefix ?? Path.GetFileNameWithoutExtension(pdfPath); + //outputFolder = String.IsNullOrEmpty(outputFolder) ? Path.GetDirectoryName(pdfPath) : outputFolder; + + quality = quality; + + Stopwatch sw = Stopwatch.StartNew(); + using (var pdfReader = new PdfReader(pdfPath)) + { + Logger.Log($"Processing[{pdfReader.NumberOfPages}]: " + pdfPath, LogLevel.Verbose); + + if (pdfReader.IsEncrypted()) + throw new ApplicationException(pdfPath + " is encrypted."); + + var pdfParser = new PdfReaderContentParser(pdfReader); + + output = ZipArchive.Create(); + { + imageStreams = new List(); + int count = 0; + Cover = true; + while (_currentPage <= pdfReader.NumberOfPages) + { + pdfParser.ProcessContent(_currentPage, this); + + _currentPage++; + count++; + if (count % 10 == 0) + { + string digit = GetPlace(count, 10).ToString(); + Logger.Log(digit, LogLevel.Verbose, false); + } + } + + + Directory.CreateDirectory(Path.GetDirectoryName(outputFolder)); + + output.SaveTo(outputFolder, CompressionType.Deflate); + + //output.SaveTo(outputPath, CompressionType.None); + + imageStreams.ForEach(i => i.Dispose()); + } + output.Dispose(); + int nop = pdfReader.NumberOfPages; + if (nop == 0) nop = 1; + long elapsed = sw.ElapsedMilliseconds; + //Logger.Log($"\r\nFinished({save.ToString("0.##")}): " + pdfPath, LogLevel.Verbose); + Logger.Log($"\r\nFinished({0.ToString("0.##")} [{elapsed}][{elapsed / nop}]): " + pdfPath, LogLevel.Verbose); + + } + + return _imageCount; + } + private Bitmap ChangePixelFormat(Bitmap orig, System.Drawing.Imaging.PixelFormat format = System.Drawing.Imaging.PixelFormat.Format24bppRgb) + { + Bitmap clone = new Bitmap(orig.Width, orig.Height, format); + + using (Graphics gr = Graphics.FromImage(clone)) + { + gr.DrawImage(orig, new Rectangle(0, 0, clone.Width, clone.Height)); + } + + return clone; + + } + + #region Implementation of IRenderListener + + public void BeginTextBlock() + { + } + + public void EndTextBlock() + { + } + + public void RenderText(TextRenderInfo renderInfo) + { + var text = renderInfo.GetText(); + } + + public void RenderImage(ImageRenderInfo renderInfo) + { + try + { + var type = renderInfo.GetType(); + var str = renderInfo.toString(); + //var ftype = renderInfo.GetFileType(); + var imageObject = renderInfo.GetImage(); + + //var imageFileName = String.Format("{0}_{1}_{2}.{3}", _outputFilePrefix, _currentPage, _imageCount, + // imageObject.GetFileType()); + var imageFileName = String.Format($"{_imageCount.ToString("0000")}"); + //var imagePath = Path.Combine(_outputFolder, imageFileName); + + //if (_overwriteExistingFiles || !File.Exists(imagePath)) + { + var imageRawBytes = imageObject.GetImageAsBytes(); + var image = imageObject.GetDrawingImage(); + + //File.WriteAllBytes(imagePath, imageRawBytes); + if (image.Width < 512 || image.Height < 512) + return; + + if (image.PixelFormat != System.Drawing.Imaging.PixelFormat.Format24bppRgb) + { + var newBits = ChangePixelFormat((Bitmap)image); + image.Dispose(); + image = newBits; + } + + var ms = new MemoryStream(); + encoder.Encode((Bitmap)image, ms, Quality); + if (!Cover && ms.Length < imageRawBytes.Length) + { + imageStreams.Add(ms); + output.AddEntry(imageFileName + ".webp", ms); + Logger.Log(".", LogLevel.Verbose, false); + } + else + { + var mso = new MemoryStream(); + mso.Write(imageRawBytes); + output.AddEntry(imageFileName + imageObject.GetFileType(), mso); + Logger.Log(Cover ? "C" : "=", LogLevel.Verbose, false); + Cover = false; + } + } + + // Subtle: Always increment even if file is not written. This ensures consistency should only some + // of a PDF file's images actually exist. + _imageCount++; + } + catch (Exception e) + { + Console.WriteLine(e); + throw; + } + + } + + #endregion // Implementation of IRenderListener + + } +} + diff --git a/ImageExtractor7.cs b/ImageExtractor7.cs new file mode 100644 index 0000000..ed1f147 --- /dev/null +++ b/ImageExtractor7.cs @@ -0,0 +1,89 @@ +using System; +using System.IO; +using System.Linq; + +using iText.Kernel.Geom; +using iText.Kernel.Pdf; +using iText.Kernel.Pdf.Xobject; +using iText.Layout; +using iText.Layout.Element; + +using java.util; +using org.apache.pdfbox.pdmodel.graphics.xobject; +using org.apache.pdfbox.pdmodel; + +namespace ComicCompressor +{ + public class ImageExtractor7 + { + public static readonly String DEST = "K:\\_COMIX\\_TOOLS_\\comic-compress-master\\bin\\Debug\\net6.0\\_NEW_PDF_\\Fan SUP 18.cbz"; + + public static readonly String SRC = "K:\\_COMIX\\_TOOLS_\\comic-compress-master\\bin\\Debug\\net6.0\\_PDF_\\Fan SUP 18.pdf"; + + public static void Main7() + { + FileInfo file = new FileInfo(DEST); + file.Directory.Create(); + + new ImageExtractor7().ManipulatePdf(DEST); + } + + protected void ManipulatePdf(String dest) + { + PdfDocument resultDoc = new PdfDocument(new PdfWriter(dest)); + PdfDocument srcDoc = new PdfDocument(new PdfReader(SRC)); + + // Assume that there is a single XObject in the source document + // and this single object is an image. + PdfDictionary pageDict = srcDoc.GetFirstPage().GetPdfObject(); + PdfDictionary pageResources = pageDict.GetAsDictionary(PdfName.Resources); + PdfDictionary pageXObjects = pageResources.GetAsDictionary(PdfName.XObject); + var ks = pageXObjects.KeySet(); + PdfName imgRef = pageXObjects.KeySet().First(); + PdfStream imgStream = pageXObjects.GetAsStream(imgRef); + PdfImageXObject imgObject = new PdfImageXObject((PdfStream)imgStream.CopyTo(resultDoc)); + Image image = new Image(imgObject); + image.ScaleToFit(14400, 14400); + image.SetFixedPosition(0, 0); + + + srcDoc.Close(); + + PageSize pageSize = new PageSize(image.GetImageScaledWidth(), image.GetImageScaledHeight()); + Document doc = new Document(resultDoc, pageSize); + doc.Add(image); + + doc.Close(); + } + + + public static void MainBox() + { + new ImageExtractor7().ManipulatePdf(DEST); + } + + protected void ManipulatePdfBox(String dest) + { + PDDocument document = null; + document = PDDocument.load(SRC); + java.util.List pages = document.getDocumentCatalog().getAllPages(); + Iterator iter = pages.iterator(); + while (iter.hasNext()) + { + PDPage page = (PDPage)iter.next(); + PDResources resources = page.getResources(); + Map pageImages = resources.getImages(); + if (pageImages != null) + { + Iterator imageIter = pageImages.keySet().iterator(); + while (imageIter.hasNext()) + { + String key = (String)imageIter.next(); + PDXObjectImage image = (PDXObjectImage)pageImages.get(key); + //image.write2OutputStream(/* some output stream */); + } + } + } + } + } +} diff --git a/Logger.cs b/Logger.cs new file mode 100644 index 0000000..6a025bb --- /dev/null +++ b/Logger.cs @@ -0,0 +1,93 @@ +using System; +using System.Drawing; +using System.IO; +using System.Runtime.CompilerServices; + +using Pastel; + +namespace ComicCompressor +{ + public class Logger + { + public LogLevel LoggingLevel { get; set; } = LogLevel.Warning; + public bool Debug = false; + + // Set to null for stdout + public string LogFile { get; set; } = null; + + public void Log(object message, LogLevel logLevel, Color? col = null) + { + Log(message.ToString(), logLevel, col:col); + } + + public void Log(string message, LogLevel logLevel,bool newLine = true, Color? col = null) + { + if (logLevel > LoggingLevel) return; + + if (LogFile == null) + { + if(newLine) + Console.WriteLine(message.Pastel(col ?? Color.FromArgb(0,204,204,204))); + else + Console.Write(message.Pastel(col ?? Color.FromArgb(0, 204, 204, 204))); + return; + } + + WriteToFile(message); + } + + public void LogError(string message, Color? col = null) + { + Log(message, LogLevel.Error, col:col); + } + + public void LogError(object message, Color? col = null) + { + Log(message, LogLevel.Error, col:col); + } + + public void LogDebug(string message, LogLevel logLevel, Color? col = null) + { + if (!Debug) return; + Log(message, logLevel,col:col); + } + + public void LogDebug(object message, LogLevel logLevel, Color? col = null) + { + LogDebug(message.ToString(), logLevel, col:col); + } + + [MethodImpl(MethodImplOptions.Synchronized)] + private void WriteToFile(string message) + { + if (LogFile == null) return; + + if (!Path.IsPathRooted(LogFile) && !LogFile.StartsWith(".")) + { + LogFile = Path.Join(".", LogFile); + } + + try + { + Directory.CreateDirectory(Path.GetDirectoryName(LogFile)); + File.AppendAllText(Path.GetFullPath(LogFile), message + "\n"); + } + catch (UnauthorizedAccessException) + { + Console.WriteLine("Error: User does not have permissions for log file: " + LogFile); + return; + } + catch (Exception) + { + Console.WriteLine("Error: could not write to log file: " + LogFile); + return; + } + } + + } + + public enum LogLevel + { + Error = 2, Warning = 3, Info = 4, Verbose = 5 + } +} diff --git a/Program.cs b/Program.cs new file mode 100644 index 0000000..9692090 --- /dev/null +++ b/Program.cs @@ -0,0 +1,552 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using System.Collections.Generic; +using McMaster.Extensions.CommandLineUtils; +using System.ComponentModel.DataAnnotations; +using System.Diagnostics; +using System.Threading; +using System.Drawing.Imaging; +using System.Drawing; +using System.Linq; +using System.Runtime.InteropServices; + +using GroupDocs.Parser; +using GroupDocs.Parser.Data; +using iText.Kernel.Pdf; + +using Imazen.WebP; + +using SharpCompress.Archives; +using SharpCompress.Archives.Zip; +using SharpCompress.Common; +using itext.pdfimage.Extensions; +using org.omg.PortableInterceptor; +using org.apache.pdfbox.util.@operator; +using java.nio; +using ImageMagick; + +namespace ComicCompressor +{ + public class Program + { + public static int Main(string[] args) => CommandLineApplication.Execute(args); + + [FileOrDirectoryExists] + [Option("-i|--input ", Description = "The file or folder to convert")] + [Required] + public string Input { get; } + + [Option("-o|--output ", + Description = + "Base path of the output (will be in output/subfolders if the recursive option is enabled), default: converted_comics")] + public string OutputFolder { get; } = "converted_comics"; + + [Option("-r|--recursive", Description = "Recursively traverse the input folder (include all subfolders)")] + public bool Recursive { get; } = false; + + [Option("-s|--skip", Description = "Skip processing file if it already exists in the output folder")] + public bool Skip { get; } = false; + + [Option("-q|--quality", Description = "Quality to use for the webp files (default: 75)")] + public int Quality { get; } = 75; + + [Option("-p|--parallel", Description = "Run in parallel, utilizing all computing resources")] + public bool IsParallel { get; } = false; + + [Option("-c|--cpu", Description = "Run in parallel, utilizing given computing resources")] + public int NumOfCpu { get; } = Environment.ProcessorCount; + + + [Option("-d|--pdf", Description = "process pdf only")] + public bool DoPdf { get; } = false; + + [Option("-j|--image", Description = "process images only")] + public bool DoImage { get; } = false; + + private Logger Logger { get; set; } + + SimpleEncoder encoder = new SimpleEncoder(); + private List imageStreams = null; + public ZipArchive output = null; + private void OnExecute() + { + + //ImageExtractor7.MainBox(); + + Stopwatch sw = Stopwatch.StartNew(); + + Logger = new Logger(); + Logger.Debug = true; + Logger.LoggingLevel = LogLevel.Verbose; + + IList files = new FileParser().ListAllFiles(Input, Recursive); + + Process(files); + + long time_query = sw.ElapsedMilliseconds; + Logger.Log($"ElapsedTime {time_query} ms", LogLevel.Verbose, true); + + Console.ReadKey(); + + } + + private void Process(IList files) + { + Compressor compressor = new Compressor(Logger); + compressor.Quality = Quality; + int filename_len = 0; + + foreach (var file in files) + { + if(file.Length > filename_len) + filename_len = file.Length; + } + + + if (!IsParallel) + { + int n = 1; + + foreach (var file in files) + { + Logger.Log($"{n,3}/{files.Count,3} - ", LogLevel.Verbose, false); + CompressTask(compressor, file, filename_len); + n++; + } + + return; + } + + ParallelOptions options = new ParallelOptions + { + MaxDegreeOfParallelism = (NumOfCpu > Environment.ProcessorCount) ? Environment.ProcessorCount : NumOfCpu + }; + + Parallel.ForEach(files, options, file => CompressTask(compressor, file, filename_len, true)); + + } + + private void CompressTask(Compressor compressor, string filename, int max_name_len = -1, Boolean isPar = false) + { + var relativePath = Path.GetRelativePath(Input, filename); + var outputPath = Path.Join(OutputFolder, Path.ChangeExtension(relativePath, "cbz")); + var outputPath_webp = Path.Join(OutputFolder, Path.ChangeExtension(relativePath, "webp")); + + if (File.Exists(outputPath) && Skip) + { + Logger.Log(outputPath, LogLevel.Verbose, true, Color.Yellow); + return; + } + if (DoImage && Skip && File.Exists(outputPath_webp)) + { + Logger.Log(outputPath_webp, LogLevel.Verbose, true, Color.Yellow); + return; + } + + if (DoPdf && filename.ToLower().EndsWith(".pdf")) + { + ExtractImageFromPDF(filename, outputPath); + //compressor.CompressPdf(filename, outputPath); + return; + } + + if (DoPdf) + return; + + if (DoImage && + filename.ToLower().EndsWith("jpg") || + filename.ToLower().EndsWith("jpeg") || + filename.ToLower().EndsWith("png")) + { + outputPath = Path.Join(OutputFolder, Path.ChangeExtension(relativePath, "webp")); + CompressImage(filename, outputPath, max_name_len,isPar); + return; + } + + //if (DoImage && + // filename.ToLower().EndsWith("psd")) + //{ + // outputPath = Path.Join(OutputFolder, Path.ChangeExtension(relativePath, "webp")); + // CompressImagePSD(filename, outputPath); + // return; + //} + + if (!filename.ToLower().EndsWith(".cbr") && !filename.ToLower().EndsWith(".cbz")) + { + Logger.Log(relativePath, LogLevel.Verbose, true, Color.LightYellow); + return; + } + + try + { + compressor.Compress(filename, outputPath); + } + catch (Exception e) + { + Logger.Log("Error processing file: " + filename, LogLevel.Error, Color.Red); + Logger.LogDebug(e, LogLevel.Error, col:Color.Red); + } + } + + #region COMPRESS_PDF + +#if false + public static string ExtractImageFromPDF(string sourcePdf) + { + iText.Kernel.Pdf.PdfReader reader = new iText.Kernel.Pdf.PdfReader(sourcePdf); + try + { + iText.Kernel.Pdf.PdfDocument document = new iText.Kernel.Pdf.PdfDocument(reader); + + for (int pageNumber + = 1; pageNumber <= document.GetNumberOfPages(); pageNumber++) + { + iText.Kernel.Pdf.PdfDictionary obj = + (iText.Kernel.Pdf.PdfDictionary)document.GetPdfObject(pageNumber); + + if (obj != null && obj.IsStream()) + { + iText.Kernel.Pdf.PdfDictionary pd + = (iText.Kernel.Pdf.PdfDictionary)obj; + if (pd.ContainsKey(iText.Kernel.Pdf.PdfName.Subtype) && pd.Get(iText.Kernel.Pdf.PdfName.Subtype).ToString() == "/Image") + { + string filter + = pd.Get(iText.Kernel.Pdf.PdfName.Filter).ToString(); + string width + = pd.Get(iText.Kernel.Pdf.PdfName.Width).ToString(); + string height + = pd.Get(iText.Kernel.Pdf.PdfName.Height).ToString(); + string bpp + = pd.Get(iText.Kernel.Pdf.PdfName.BitsPerComponent).ToString(); + string extent + = "."; + byte[] img + = pd.Get(iText.Kernel.Pdf.PdfName.Image); + Debug.WriteLine(filter); + continue; + switch (filter) + { + case "/FlateDecode": + byte[] arr + = FlateDecodeFilter.FlateDecode(img, true); + Bitmap bmp + = new Bitmap(Int32.Parse(width), Int32.Parse(height), PixelFormat.Format24bppRgb); + BitmapData bmd + = bmp.LockBits(new Rectangle(0, 0, Int32.Parse(width), Int32.Parse(height)), ImageLockMode.WriteOnly, + PixelFormat.Format24bppRgb); + Marshal.Copy(arr, 0, bmd.Scan0, arr.Length); + bmp.UnlockBits(bmd); + bmp.Save("d:\\pdf\\bmp1.png", ImageFormat.Png); + break; + case "/CCITTFaxDecode": + break; + default: + break; + } + } + } + } + } + catch + { + throw; + } + return ""; + } +#elif false + public static string ExtractImageFromPDF(string sourcePdf) + { + // Create an instance of Parser class + using (Parser parser = new Parser(sourcePdf)) + { + // Extract images + var di = parser.GetDocumentInfo(); + + for (int p = 0; p < di.PageCount; p++) + { + + IEnumerable images = parser.GetImages(p); + //IEnumerable images = parser.GetImages(); + // Check if image extraction is supported + if (images == null) + { + Console.WriteLine("Images extraction isn’t supported"); + return ""; + } + + int counter = 1; + // Iterate over images + foreach (PageImageArea image in images) + { + // Save each image + Image.FromStream(image.GetImageStream()) + .Save(string.Format($"{p.ToString("0000")}_{counter++.ToString("0000")}.jpg"), ImageFormat.Jpeg); + } + } + } + return ""; + } +#else + public int GetPlace(int value, int place) + { + return ((value % (place * 10)) - (value % place)) / place; + } + private bool Cover = true; + int _currentPage = 1; + int _imageCount = 0; + public void ExtractImageFromPDF(string sourcePdf, string outputFolder) + { + + long input_length = new System.IO.FileInfo(sourcePdf).Length; + var pdf = File.Open(sourcePdf, FileMode.Open); + + var reader = new PdfReader(pdf); + var pdfDocument = new PdfDocument(reader); + int numOfPages = pdfDocument.GetNumberOfPages(); + //var bitmaps = pdfDocument.ConvertToBitmaps(); + + Logger.Log($"Processing[{numOfPages}]: " + outputFolder, LogLevel.Verbose); + + output = ZipArchive.Create(); + imageStreams = new List(); + int count = 0; + Cover = true; + Stopwatch sw = Stopwatch.StartNew(); + for (int p = 1; p <= numOfPages; p++) + { + PdfPage page = pdfDocument.GetPage(p); + //var bitmap = page.ConvertPageToBitmap(); + MemoryStream jpg = (MemoryStream)page.ConvertPageToJpg(); + + //bitmap.Save(Path.Combine($"wave-{DateTime.Now.Ticks}.png"), ImageFormat.Png); + ProcessPdfImage(jpg); + //jpg.Dispose(); + //page.Flush(); + + _currentPage++; + count++; + if (count % 10 == 0) + { + string digit = GetPlace(count, 10).ToString(); + Logger.Log(digit, LogLevel.Verbose, false); + } + } + Directory.CreateDirectory(Path.GetDirectoryName(outputFolder)); + + output.SaveTo(outputFolder, CompressionType.Deflate); + + //output.SaveTo(outputPath, CompressionType.None); + + imageStreams.ForEach(i => i.Dispose()); + + output.Dispose(); + + long output_length = new System.IO.FileInfo(outputFolder).Length; + + double save = 100 * (double)output_length / (double)input_length; + if (numOfPages == 0) numOfPages = 1; + long elapsed = sw.ElapsedMilliseconds; + Logger.Log($"\r\nFinished({save.ToString("0.##")}) [{elapsed}][{elapsed / numOfPages}]: " + outputFolder, LogLevel.Verbose); + + } + private Bitmap ChangePixelFormat(Bitmap orig, System.Drawing.Imaging.PixelFormat format = System.Drawing.Imaging.PixelFormat.Format24bppRgb) + { + Bitmap clone = new Bitmap(orig.Width, orig.Height, format); + + using (Graphics gr = Graphics.FromImage(clone)) + { + gr.DrawImage(orig, new System.Drawing.Rectangle(0, 0, clone.Width, clone.Height)); + } + + return clone; + + } + + public static byte[] ImageToByte(Image img) + { + ImageConverter converter = new ImageConverter(); + return (byte[])converter.ConvertTo(img, typeof(byte[])); + } + public void ProcessPdfImage(MemoryStream jpg) + { + Stopwatch sw = Stopwatch.StartNew(); + try + { + var image = new Bitmap(jpg); + //var imageFileName = String.Format("{0}_{1}_{2}.{3}", _outputFilePrefix, _currentPage, _imageCount, + // imageObject.GetFileType()); + var imageFileName = String.Format($"{_imageCount.ToString("0000")}"); + //var imagePath = Path.Combine(_outputFolder, imageFileName); + + //if (_overwriteExistingFiles || !File.Exists(imagePath)) + { + //Logger.Log($"0 [{image.Width}][{image.Height}]" + sw.ElapsedMilliseconds.ToString(), LogLevel.Verbose); + + //var imageRawBytes = ImageToByte(image); + //var image = imageObject.GetDrawingImage(); + + //Logger.Log("1 "+sw.ElapsedMilliseconds.ToString(), LogLevel.Verbose); + + //File.WriteAllBytes(imagePath, imageRawBytes); + if (image.Width < 512 || image.Height < 512) + return; + + if (image.PixelFormat != System.Drawing.Imaging.PixelFormat.Format24bppRgb) + { + var newBits = ChangePixelFormat((Bitmap)image); + image.Dispose(); + image = newBits; + } + //Logger.Log("2 " + sw.ElapsedMilliseconds.ToString(), LogLevel.Verbose); + + var ms = new MemoryStream(); + encoder.Encode((Bitmap)image, ms, Quality); + //Logger.Log("3 " + sw.ElapsedMilliseconds.ToString(), LogLevel.Verbose); + + if (!Cover && ms.Length < jpg.Length) + { + imageStreams.Add(ms); + output.AddEntry(imageFileName + ".webp", ms); + Logger.Log(".", LogLevel.Verbose, false); + //Logger.Log("4 " + sw.ElapsedMilliseconds.ToString(), LogLevel.Verbose); + + } + else + { + //var mso = new MemoryStream(); + //mso.Write(imageRawBytes); + // output.AddEntry(imageFileName + imageObject.GetFileType(), mso); + imageStreams.Add(ms); + output.AddEntry(imageFileName + ".jpg", jpg); + Logger.Log(Cover ? "C" : "=", LogLevel.Verbose, false); + Cover = false; + } + } + + // Subtle: Always increment even if file is not written. This ensures consistency should only some + // of a PDF file's images actually exist. + _imageCount++; + } + catch (Exception e) + { + Console.WriteLine(e); + throw; + } + + //Logger.Log(sw.ElapsedMilliseconds.ToString(), LogLevel.Verbose); + } +#endif + +#endregion + + #region COMPRESS_IMAGE + public void CompressImage(string imagefile, string outputFolder, int max_name_len = -1, Boolean isPar = false) + { + string name_print = imagefile; + if(max_name_len > 0) + name_print = imagefile.PadRight(max_name_len); + if(!isPar) + Logger.Log(name_print, LogLevel.Verbose, false, col:Color.LightBlue); + + try + { + Directory.CreateDirectory(Path.GetDirectoryName(outputFolder)); + + var encoder = new SimpleEncoder(); + + Bitmap mBitmap; + long length_in = 0; + FileStream outStream = new FileStream(outputFolder, FileMode.Create); + using (Stream BitmapStream = System.IO.File.Open(imagefile, System.IO.FileMode.Open)) + { + length_in = BitmapStream.Length; + Image img = Image.FromStream(BitmapStream); + + mBitmap = new Bitmap(img); + + IntPtr result; + long length = 0; + encoder.Encode(mBitmap, outStream, 70); + + outStream.Flush(); + outStream.Close(); + //encoder.Encode(mBitmap, 70, out result, out length); + //UnmanagedMemoryStream ustream = new UnmanagedMemoryStream((byte*)result, length); + //ustream.CopyTo(outStream); + //ustream.Close(); + //outStrea.Close(); + } + + FileInfo finfo = new FileInfo(outputFolder); + long length_out = finfo.Length; + if (isPar) + Logger.Log($"{name_print} - IN[{length_in,10}] OUT[{length_out,10}] %[{(100.0 * length_out / (double)length_in),5:##.00}]", LogLevel.Verbose, true,col:Color.Green); + else + Logger.Log($" - IN[{length_in,10}] OUT[{length_out,10}] %[{(100.0 * length_out/(double)length_in),5:##.00}]", LogLevel.Verbose, true, col:Color.LightBlue); + } + catch(Exception e) + { + Logger.LogError(e, col: Color.Red); + } + + } + + + public void CompressImagePSD(string imagefile, string outputFolder) + { + Logger.Log(imagefile, LogLevel.Verbose, false); + + try + { + Directory.CreateDirectory(Path.GetDirectoryName(outputFolder)); + + var encoder = new SimpleEncoder(); + + Bitmap mBitmap; + long length_in = 0; + FileStream outStream = new FileStream(outputFolder, FileMode.Create); + using (Stream BitmapStream = System.IO.File.Open(imagefile, System.IO.FileMode.Open)) + { + length_in = BitmapStream.Length; + + //using var imageFromFile = new MagickImage(BitmapStream); + using (MagickImageCollection magickImages = new MagickImageCollection(BitmapStream)) + { + foreach (var image in magickImages) + { + Image img = image.ToBitmap(); + + + //Image img = Image.FromStream(BitmapStream); + //Image img = imageFromFile.ToBitmap(); + mBitmap = new Bitmap(img); + + IntPtr result; + long length = 0; + encoder.Encode(mBitmap, outStream, 70); + } + } + + outStream.Flush(); + outStream.Close(); + //encoder.Encode(mBitmap, 70, out result, out length); + //UnmanagedMemoryStream ustream = new UnmanagedMemoryStream((byte*)result, length); + //ustream.CopyTo(outStream); + //ustream.Close(); + //outStrea.Close(); + } + + FileInfo finfo = new FileInfo(outputFolder); + long length_out = finfo.Length; + Logger.Log($" - IN[{length_in}] OUT[{length_out}] %[{100.0 * length_out / (double)length_in}]", LogLevel.Verbose, true); + } + catch (Exception e) + { + Logger.LogError(e); + } + + } + #endregion + } +} diff --git a/README.md b/README.md index 973c453..779c5a3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,25 @@ -# comic_img-compress +#comic-img-compress + +**#** About +A simple command line interface to change .cbr and .cbz comic files with png or jpeg images to webp, improving compression markedly +The tool is able to convert also single jpg or png images to webp + +# Setup +This project uses [libwebp-net](https://github.com/imazen/libwebp-net), in order to use download the libwebp.dll from [here](https://s3.amazonaws.com/resizer-dynamic-downloads/webp/0.5.2/x86_64/libwebp.dll) and add to the project root + +# Running +Run `dotnet run -- [args]` +`dotnet run -- --help` for help +``` + -i|--input The file or folder to convert + -o|--output Base path of the output (will be in output/subfolders if the recursive option is enabled), default: converted_comics + -r|--recursive Recursively traverse the input folder (include all subfolders) + -s|--skip Skip processing file if it already exists in the output folder + -q|--quality Quality to use for the webp files (default: 75) + -p|--parallel Run in parallel, utilizing all computing resources + -?|-h|--help Show help information + ``` + +# Building +Run `dotnet publish -r win-x64 -c Release` -.NET console tool to compress CBZ, CBR archives with jpeg or png files to CBZ with WEBP converted images -The tool is able to convert also single jpg or png images to webp