using System; using System.IO; using System.Threading.Tasks; using System.Collections.Generic; using McMaster.Extensions.CommandLineUtils; using System.ComponentModel.DataAnnotations; using System.Diagnostics; using System.Threading; using System.Drawing.Imaging; using System.Drawing; using System.Linq; using System.Runtime.InteropServices; using GroupDocs.Parser; using GroupDocs.Parser.Data; using iText.Kernel.Pdf; using Imazen.WebP; using SharpCompress.Archives; using SharpCompress.Archives.Zip; using SharpCompress.Common; using itext.pdfimage.Extensions; using org.omg.PortableInterceptor; using org.apache.pdfbox.util.@operator; using java.nio; using ImageMagick; using javax.xml.transform; using System.Collections.Concurrent; using iText.Barcodes.Dmcode; using ZstdSharp; namespace ComicCompressor { public class Program { public static int Main(string[] args) => CommandLineApplication.Execute(args); #region INPUT_FLAGS [FileOrDirectoryExists] [Option("-i|--input ", Description = "The file or folder to convert")] [Required] public string Input { get; } [Option("-o|--output ", Description = "Base path of the output (will be in output/subfolders if the recursive option is enabled), default: converted_comics")] public string OutputFolder { get; } = "converted_comics"; [Option("-r|--recursive", Description = "Recursively traverse the input folder (include all subfolders)")] public bool Recursive { get; } = false; [Option("-s|--skip", Description = "Skip processing file if it already exists in the output folder")] public bool Skip { get; } = false; [Option("-q|--quality", Description = "Quality to use for the webp files (default: 75)")] public int Quality { get; } = 75; [Option("-p|--parallel", Description = "Run in parallel, utilizing all computing resources")] public bool IsParallel { get; } = false; [Option("-c|--cpu", Description = "Run in parallel, utilizing given computing resources")] public int NumOfCpu { get; } = Environment.ProcessorCount; [Option("-d|--pdf", Description = "process pdf only")] public bool DoPdf { get; } = false; [Option("-j|--image", Description = "process images only")] public bool DoImage { get; } = false; #endregion #region STATISTICS public class file_stat { public string Filename { get; set; } public file_stat(string filename) { Filename = filename; } public long Uncompressed_file_len { get; set; } public long Compressed_file_len { get; set; } } static Dictionary file_statistics = new Dictionary(); public static Dictionary File_statistics { get => file_statistics; set => file_statistics = value; } #endregion private Logger Logger { get; set; } SimpleEncoder encoder = new SimpleEncoder(); private List imageStreams = null; public ZipArchive output = null; static Random randomGen = new Random(); static KnownColor[] names = (KnownColor[])Enum.GetValues(typeof(KnownColor)); // static KnownColor randomColorName = names[randomGen.Next(names.Length)]; // static Color randomColor = Color.FromKnownColor(randomColorName); public Color[] myColors = null; private void OnExecute() { File_statistics.Clear(); myColors = new Color[names.Count()]; for(int i = 0; i< myColors.Length; i++) { KnownColor randomColorName = names[randomGen.Next(names.Length)]; myColors[i] = Color.FromKnownColor(randomColorName); } //ImageExtractor7.MainBox(); Stopwatch sw = Stopwatch.StartNew(); Logger = new Logger(); Logger.Debug = true; Logger.LoggingLevel = LogLevel.Verbose; IList files = new FileParser().ListAllFiles(Input, Recursive); Process(files); long time_query = sw.ElapsedMilliseconds; Logger.Log($"ElapsedTime {time_query} ms", LogLevel.Verbose, true); Console.ReadKey(); } private void Process(IList files) { Compressor compressor = new Compressor(Logger); compressor.Quality = Quality; int filename_len = 0; foreach (var file in files) { File_statistics.Add(file, new file_stat(file)); if(file.Length > filename_len) filename_len = file.Length; } if (!IsParallel) { int n = 1; foreach (var file in files) { Logger.Log($"{n,3}/{files.Count,3} - ", LogLevel.Verbose, false); CompressTask(compressor, file, filename_len); n++; } long tot_uncomp = 0; long tot_comp = 0; foreach (var file in file_statistics) { long uncomp = file.Value.Uncompressed_file_len; long comp = file.Value.Compressed_file_len; if (uncomp != 0 && comp != 0) { Logger.Log($"{file.Value.Filename} U[{uncomp}] C[{comp}]", LogLevel.Verbose, true, Color.Pink); tot_uncomp += uncomp; tot_comp += comp; } } if (tot_uncomp != 0) { double save = 100 * (double)tot_comp / (double)tot_uncomp; Logger.Log($"TU[{tot_uncomp}] TC[{tot_comp}] R%[{save,5:##.00}] S[{(tot_uncomp-tot_comp)/1024.0/1024.0,5:##.00} MB]", LogLevel.Verbose, true, Color.Pink); } return; } ParallelOptions options = new ParallelOptions { MaxDegreeOfParallelism = (NumOfCpu > Environment.ProcessorCount) ? Environment.ProcessorCount : NumOfCpu }; #if true ParallelQuery source = Partitioner.Create(files) .AsParallel() .AsOrdered(); Parallel.ForEach(source, options, (file, state, index) => CompressTask(compressor, file, filename_len, true, state, index, files.Count)); #else Parallel.ForEach(files, options, (file, state, index) => CompressTask(compressor, file, filename_len, true, state, index+1, files.Count)); #endif foreach (var file in file_statistics) { Logger.Log($"{file.Value.Filename} U[{file.Value.Uncompressed_file_len}] C[{file.Value.Compressed_file_len}]", LogLevel.Verbose, true, Color.Pink); } } private void CompressTask(Compressor compressor, string filename, int max_name_len = -1, Boolean isPar = false, ParallelLoopState state = null, long index = 0, long total = 0) { var relativePath = Path.GetRelativePath(Input, filename); var outputPath = Path.Join(OutputFolder, Path.ChangeExtension(relativePath, "cbz")); var outputPath_webp = Path.Join(OutputFolder, Path.ChangeExtension(relativePath, "webp")); int digits = 1+(int)Math.Floor(Math.Log((Double)total) / Math.Log(10.0)); string print_line_header = ""; switch (digits) { case 1: print_line_header = $"{index,1}/{total,1} - "; break; case 2: print_line_header = $"{index,2}/{total,2} - "; break; case 3: print_line_header = $"{index,3}/{total,3} - "; break; case 4: print_line_header = $"{index,4}/{total,4} - "; break; case 5: print_line_header = $"{index,5}/{total,5} - "; break; } if (File.Exists(outputPath) && Skip) { long lengthin = new System.IO.FileInfo(filename).Length; long lengthout = new System.IO.FileInfo(outputPath).Length; Program.file_stat fs = null; Program.File_statistics.TryGetValue(filename, out fs); if (fs != null) { fs.Uncompressed_file_len = lengthin; fs.Compressed_file_len = lengthout; Program.File_statistics.TryAdd(filename, fs); } Logger.Log(outputPath, LogLevel.Verbose, true, Color.Yellow); return; } if (DoImage && Skip && File.Exists(outputPath_webp)) { long lengthin = new System.IO.FileInfo(filename).Length; long lengthout = new System.IO.FileInfo(outputPath).Length; Program.file_stat fs = null; Program.File_statistics.TryGetValue(filename, out fs); if (fs != null) { fs.Uncompressed_file_len = lengthin; fs.Compressed_file_len = lengthout; Program.File_statistics.TryAdd(filename, fs); } Logger.Log(outputPath_webp, LogLevel.Verbose, true, Color.Yellow); return; } if (DoPdf && filename.ToLower().EndsWith(".pdf")) { ExtractImageFromPDF(compressor,filename, outputPath); long lengthin = new System.IO.FileInfo(filename).Length; long lengthout = new System.IO.FileInfo(outputPath).Length; Program.file_stat fs = null; Program.File_statistics.TryGetValue(filename, out fs); if (fs != null) { fs.Uncompressed_file_len = lengthin; fs.Compressed_file_len = lengthout; Program.File_statistics.TryAdd(filename, fs); } //compressor.CompressPdf(filename, outputPath); return; } //if (DoPdf) // return; if (DoImage && filename.ToLower().EndsWith("jpg") || filename.ToLower().EndsWith("jpeg") || filename.ToLower().EndsWith("tif") || filename.ToLower().EndsWith("png")) { outputPath = Path.Join(OutputFolder, Path.ChangeExtension(relativePath, "webp")); CompressImage(filename, outputPath, max_name_len,isPar, print_line_header); long lengthin = new System.IO.FileInfo(filename).Length; long lengthout = new System.IO.FileInfo(outputPath).Length; Program.file_stat fs = null; Program.File_statistics.TryGetValue(filename, out fs); if (fs != null) { fs.Uncompressed_file_len = lengthin; fs.Compressed_file_len = lengthout; Program.File_statistics.TryAdd(filename, fs); } return; } //if (DoImage && // filename.ToLower().EndsWith("psd")) //{ // outputPath = Path.Join(OutputFolder, Path.ChangeExtension(relativePath, "webp")); // CompressImagePSD(filename, outputPath); // return; //} if (!filename.ToLower().EndsWith(".cbr") && !filename.ToLower().EndsWith(".cbz")) { Logger.Log(relativePath, LogLevel.Verbose, true, Color.LightYellow); return; } try { compressor.Compress(filename, outputPath); } catch (Exception e) { Logger.Log("Error processing file: " + filename, LogLevel.Error, Color.Red); Logger.LogDebug(e, LogLevel.Error, col:Color.Red); } } #region COMPRESS_PDF #if false public static string ExtractImageFromPDF(string sourcePdf) { iText.Kernel.Pdf.PdfReader reader = new iText.Kernel.Pdf.PdfReader(sourcePdf); try { iText.Kernel.Pdf.PdfDocument document = new iText.Kernel.Pdf.PdfDocument(reader); for (int pageNumber = 1; pageNumber <= document.GetNumberOfPages(); pageNumber++) { iText.Kernel.Pdf.PdfDictionary obj = (iText.Kernel.Pdf.PdfDictionary)document.GetPdfObject(pageNumber); if (obj != null && obj.IsStream()) { iText.Kernel.Pdf.PdfDictionary pd = (iText.Kernel.Pdf.PdfDictionary)obj; if (pd.ContainsKey(iText.Kernel.Pdf.PdfName.Subtype) && pd.Get(iText.Kernel.Pdf.PdfName.Subtype).ToString() == "/Image") { string filter = pd.Get(iText.Kernel.Pdf.PdfName.Filter).ToString(); string width = pd.Get(iText.Kernel.Pdf.PdfName.Width).ToString(); string height = pd.Get(iText.Kernel.Pdf.PdfName.Height).ToString(); string bpp = pd.Get(iText.Kernel.Pdf.PdfName.BitsPerComponent).ToString(); string extent = "."; byte[] img = pd.Get(iText.Kernel.Pdf.PdfName.Image); Debug.WriteLine(filter); continue; switch (filter) { case "/FlateDecode": byte[] arr = FlateDecodeFilter.FlateDecode(img, true); Bitmap bmp = new Bitmap(Int32.Parse(width), Int32.Parse(height), PixelFormat.Format24bppRgb); BitmapData bmd = bmp.LockBits(new Rectangle(0, 0, Int32.Parse(width), Int32.Parse(height)), ImageLockMode.WriteOnly, PixelFormat.Format24bppRgb); Marshal.Copy(arr, 0, bmd.Scan0, arr.Length); bmp.UnlockBits(bmd); bmp.Save("d:\\pdf\\bmp1.png", ImageFormat.Png); break; case "/CCITTFaxDecode": break; default: break; } } } } } catch { throw; } return ""; } #elif false public static string ExtractImageFromPDF(string sourcePdf) { // Create an instance of Parser class using (Parser parser = new Parser(sourcePdf)) { // Extract images var di = parser.GetDocumentInfo(); for (int p = 0; p < di.PageCount; p++) { IEnumerable images = parser.GetImages(p); //IEnumerable images = parser.GetImages(); // Check if image extraction is supported if (images == null) { Console.WriteLine("Images extraction isn’t supported"); return ""; } int counter = 1; // Iterate over images foreach (PageImageArea image in images) { // Save each image Image.FromStream(image.GetImageStream()) .Save(string.Format($"{p.ToString("0000")}_{counter++.ToString("0000")}.jpg"), ImageFormat.Jpeg); } } } return ""; } #elif false public int GetPlace(int value, int place) { return ((value % (place * 10)) - (value % place)) / place; } private bool Cover = true; int _currentPage = 1; int _imageCount = 0; public void ExtractImageFromPDF(string sourcePdf, string outputFolder) { long input_length = new System.IO.FileInfo(sourcePdf).Length; var pdf = File.Open(sourcePdf, FileMode.Open); var reader = new PdfReader(pdf); var pdfDocument = new PdfDocument(reader); int numOfPages = pdfDocument.GetNumberOfPages(); //var bitmaps = pdfDocument.ConvertToBitmaps(); Logger.Log($"Processing[{numOfPages}]: " + outputFolder, LogLevel.Verbose); output = ZipArchive.Create(); imageStreams = new List(); int count = 0; Cover = true; Stopwatch sw = Stopwatch.StartNew(); for (int p = 1; p <= numOfPages; p++) { PdfPage page = pdfDocument.GetPage(p); //var bitmap = page.ConvertPageToBitmap(); MemoryStream jpg = (MemoryStream)page.ConvertPageToJpg(); //bitmap.Save(Path.Combine($"wave-{DateTime.Now.Ticks}.png"), ImageFormat.Png); ProcessPdfImage(jpg); //jpg.Dispose(); //page.Flush(); _currentPage++; count++; if (count % 10 == 0) { string digit = GetPlace(count, 10).ToString(); Logger.Log(digit, LogLevel.Verbose, false); } } Directory.CreateDirectory(Path.GetDirectoryName(outputFolder)); output.SaveTo(outputFolder, CompressionType.Deflate); //output.SaveTo(outputPath, CompressionType.None); imageStreams.ForEach(i => i.Dispose()); output.Dispose(); long output_length = new System.IO.FileInfo(outputFolder).Length; double save = 100 * (double)output_length / (double)input_length; if (numOfPages == 0) numOfPages = 1; long elapsed = sw.ElapsedMilliseconds; Logger.Log($"\r\nFinished({save.ToString("0.##")}) [{elapsed}][{elapsed / numOfPages}]: " + outputFolder, LogLevel.Verbose); } private Bitmap ChangePixelFormat(Bitmap orig, System.Drawing.Imaging.PixelFormat format = System.Drawing.Imaging.PixelFormat.Format24bppRgb) { Bitmap clone = new Bitmap(orig.Width, orig.Height, format); using (Graphics gr = Graphics.FromImage(clone)) { gr.DrawImage(orig, new System.Drawing.Rectangle(0, 0, clone.Width, clone.Height)); } return clone; } public static byte[] ImageToByte(Image img) { ImageConverter converter = new ImageConverter(); return (byte[])converter.ConvertTo(img, typeof(byte[])); } public void ProcessPdfImage(MemoryStream jpg) { Stopwatch sw = Stopwatch.StartNew(); try { var image = new Bitmap(jpg); //var imageFileName = String.Format("{0}_{1}_{2}.{3}", _outputFilePrefix, _currentPage, _imageCount, // imageObject.GetFileType()); var imageFileName = String.Format($"{_imageCount.ToString("0000")}"); //var imagePath = Path.Combine(_outputFolder, imageFileName); //if (_overwriteExistingFiles || !File.Exists(imagePath)) { //Logger.Log($"0 [{image.Width}][{image.Height}]" + sw.ElapsedMilliseconds.ToString(), LogLevel.Verbose); //var imageRawBytes = ImageToByte(image); //var image = imageObject.GetDrawingImage(); //Logger.Log("1 "+sw.ElapsedMilliseconds.ToString(), LogLevel.Verbose); //File.WriteAllBytes(imagePath, imageRawBytes); if (image.Width < 512 || image.Height < 512) return; if (image.PixelFormat != System.Drawing.Imaging.PixelFormat.Format24bppRgb) { var newBits = ChangePixelFormat((Bitmap)image); image.Dispose(); image = newBits; } //Logger.Log("2 " + sw.ElapsedMilliseconds.ToString(), LogLevel.Verbose); var ms = new MemoryStream(); encoder.Encode((Bitmap)image, ms, Quality); //Logger.Log("3 " + sw.ElapsedMilliseconds.ToString(), LogLevel.Verbose); if (!Cover && ms.Length < jpg.Length) { imageStreams.Add(ms); output.AddEntry(imageFileName + ".webp", ms); Logger.Log(".", LogLevel.Verbose, false); //Logger.Log("4 " + sw.ElapsedMilliseconds.ToString(), LogLevel.Verbose); } else { //var mso = new MemoryStream(); //mso.Write(imageRawBytes); // output.AddEntry(imageFileName + imageObject.GetFileType(), mso); imageStreams.Add(ms); output.AddEntry(imageFileName + ".jpg", jpg); Logger.Log(Cover ? "C" : "=", LogLevel.Verbose, false); Cover = false; } } // Subtle: Always increment even if file is not written. This ensures consistency should only some // of a PDF file's images actually exist. _imageCount++; } catch (Exception e) { Console.WriteLine(e); throw; } //Logger.Log(sw.ElapsedMilliseconds.ToString(), LogLevel.Verbose); } #else public void ExtractImageFromPDF(Compressor compressor, string sourcePdf, string outputFolder) { //pdfimages.exe -j "Dragon Ball Full Color [Volume 1].pdf" "Dragon Ball Full Color [Volume 1]\img" //7za a -r -mx0 -tzip "Dragon Ball Full Color [Volume 1]" "Dragon Ball Full Color [Volume 1]\*" string subfolder = Path.GetFileNameWithoutExtension(sourcePdf); string folder = Path.GetDirectoryName(sourcePdf); string tempfolder = folder + "\\" + subfolder; Directory.CreateDirectory(tempfolder); Logger.Log(sourcePdf, LogLevel.Verbose, true, Color.Green); ProcessStartInfo startInfo_pdfimages = new ProcessStartInfo("pdfimages.exe"); startInfo_pdfimages.Arguments = $@" -j ""{sourcePdf}"" ""{tempfolder}\img"""; startInfo_pdfimages.UseShellExecute = false; Process pdfimages = System.Diagnostics.Process.Start(startInfo_pdfimages); pdfimages.WaitForExit(); string zipFile = tempfolder + ".zip"; ProcessStartInfo startInfo_7za = new ProcessStartInfo("7za.exe"); startInfo_7za.Arguments = $@" a -r -mx0 -tzip -sdel ""{zipFile}"" ""{tempfolder}\*"" > $null 2>&1"; startInfo_7za.UseShellExecute = false; startInfo_7za.RedirectStandardOutput = true; Process zip7za = System.Diagnostics.Process.Start(startInfo_7za); zip7za.WaitForExit(); Directory.Delete(tempfolder); if (File.Exists(outputFolder) && Skip) { Logger.Log(outputFolder, LogLevel.Verbose, true, Color.Yellow); return; } try { compressor.Compress(zipFile, outputFolder); } catch (Exception e) { Logger.Log("Error processing file: " + zipFile, LogLevel.Error, Color.Red); Logger.LogDebug(e, LogLevel.Error, col: Color.Red); } } #endif #endregion #region COMPRESS_IMAGE public void CompressImage(string imagefile, string outputFolder, int max_name_len = -1, Boolean isPar = false, string print_line_header = "") { string name_print = imagefile; if(max_name_len > 0) name_print = imagefile.PadRight(max_name_len); if(!isPar) Logger.Log(print_line_header+name_print, LogLevel.Verbose, false, col:Color.LightBlue); try { Directory.CreateDirectory(Path.GetDirectoryName(outputFolder)); var encoder = new SimpleEncoder(); Bitmap mBitmap; long length_in = 0; FileStream outStream = new FileStream(outputFolder, FileMode.Create); using (Stream BitmapStream = System.IO.File.Open(imagefile, System.IO.FileMode.Open)) { length_in = BitmapStream.Length; Image img = Image.FromStream(BitmapStream); mBitmap = new Bitmap(img); IntPtr result; long length = 0; encoder.Encode(mBitmap, outStream, 70); outStream.Flush(); outStream.Close(); //encoder.Encode(mBitmap, 70, out result, out length); //UnmanagedMemoryStream ustream = new UnmanagedMemoryStream((byte*)result, length); //ustream.CopyTo(outStream); //ustream.Close(); //outStrea.Close(); } FileInfo finfo = new FileInfo(outputFolder); long length_out = finfo.Length; if (isPar) { #if false Logger.Log($"{print_line_header}[{Environment.CurrentManagedThreadId}]{name_print} - IN[{length_in,10}] OUT[{length_out,10}] %[{(100.0 * length_out / (double)length_in),5:##.00}]", LogLevel.Verbose, true, col: Color.Green); #else lock (Logger) { Logger.Log($"{print_line_header}[{Environment.CurrentManagedThreadId,2}] ", LogLevel.Verbose, false, col: myColors[Environment.CurrentManagedThreadId]); Logger.Log($"{name_print} - IN[{length_in,10}] OUT[{length_out,10}] %[{(100.0 * length_out / (double)length_in),5:##.00}]", LogLevel.Verbose, true, col: Color.Green); } #endif } else Logger.Log($" - IN[{length_in,10}] OUT[{length_out,10}] %[{(100.0 * length_out / (double)length_in),5:##.00}]", LogLevel.Verbose, true, col: Color.LightBlue); } catch(Exception e) { lock (Logger) { if (isPar) { Logger.Log($"{print_line_header}[{Environment.CurrentManagedThreadId}]{name_print} ERROR", LogLevel.Verbose, true, col: Color.Red); } else { Logger.Log($" ERROR", LogLevel.Verbose, true, col: Color.Red); } Logger.LogError(e, col: Color.Red); } } } public void CompressImagePSD(string imagefile, string outputFolder) { Logger.Log(imagefile, LogLevel.Verbose, false); try { Directory.CreateDirectory(Path.GetDirectoryName(outputFolder)); var encoder = new SimpleEncoder(); Bitmap mBitmap; long length_in = 0; FileStream outStream = new FileStream(outputFolder, FileMode.Create); using (Stream BitmapStream = System.IO.File.Open(imagefile, System.IO.FileMode.Open)) { length_in = BitmapStream.Length; //using var imageFromFile = new MagickImage(BitmapStream); using (MagickImageCollection magickImages = new MagickImageCollection(BitmapStream)) { foreach (var image in magickImages) { Image img = image.ToBitmap(); //Image img = Image.FromStream(BitmapStream); //Image img = imageFromFile.ToBitmap(); mBitmap = new Bitmap(img); IntPtr result; long length = 0; encoder.Encode(mBitmap, outStream, 70); } } outStream.Flush(); outStream.Close(); //encoder.Encode(mBitmap, 70, out result, out length); //UnmanagedMemoryStream ustream = new UnmanagedMemoryStream((byte*)result, length); //ustream.CopyTo(outStream); //ustream.Close(); //outStrea.Close(); } FileInfo finfo = new FileInfo(outputFolder); long length_out = finfo.Length; Logger.Log($" - IN[{length_in}] OUT[{length_out}] %[{100.0 * length_out / (double)length_in}]", LogLevel.Verbose, true); } catch (Exception e) { Logger.LogError(e); } } #endregion } }