RepoFullDownloader/RepoFullDownloader-Core/Program.cs

using ICSharpCode.SharpZipLib.BZip2;
using ICSharpCode.SharpZipLib.GZip;

using RepoFullDownloader;

using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Text.Json;
using System.Threading;

namespace RepoFullDownloader_Core
{
    class Program
    {
        private static Options options = new Options();

        static void Main(string[] args)
        {
            Console.WriteLine("RepoFullDownloader by KawaiiZenbo");

            // Initial Checks
            if (!Directory.Exists("./output/"))
            {
                Directory.CreateDirectory("./output/");
            }
            // Load Options from 'options.json'
            string optionsJson = File.ReadAllText("./options.json");
            options = JsonSerializer.Deserialize<Options>(optionsJson);
            if (args.Length != 0)
            {
                string url = args[0];
                if(!args[0].StartsWith("https://") && !args[0].StartsWith("http://"))
                {
                    url = "http://" + url;
                }
                if (!args[0].EndsWith("/"))
                {
                    url += "/";
                }
                try
                {
                    DownloadRepo(url, options.originalFilenames);
                }
                catch(Exception)
                {
                    Console.WriteLine("No APT repo was found, trying as Installer repo");
                    DownloadInstallerRepo(url);
                }
            }
            else
            {
                if (!File.Exists("./options.json"))
                {
                    Console.WriteLine("Could not find options.json");
                    Console.WriteLine("Generating example...");
                    // generate example options
                    File.WriteAllText("./options.json", JsonSerializer.Serialize(new Options()));
                    return;
                }

                foreach (Repo r in options.repos)
                {
                    if (r.isInstaller)
                    {
                        DownloadInstallerRepo(r.url);
                    }
                    else
                    {
                        DownloadRepo(r.url, options.originalFilenames);
                    }
                }
            }

            // this is the part where it is over
            Console.WriteLine("done :)");
        }

        static void DownloadRepo(string link, bool keepOg)
        {
            if (!link.StartsWith("https://") && !link.StartsWith("http://"))
            {
                link = "http://" + link;
            }
            if (!link.EndsWith("/"))
            {
                link += "/";
            }
            string cleanLink = link.TrimEnd('/').Replace("http://", "").Replace("https://", "").Replace("/", "_").Replace(":", "_");
            Directory.CreateDirectory($"./output/{cleanLink}");
            WebClient webClient = new WebClient();
            // headers because some repos are 'interesting'
            webClient.Headers.Add("X-Machine", "iPod4,1");
            webClient.Headers.Add("X-Unique-ID", "0000000000000000000000000000000000000000");
            webClient.Headers.Add("X-Firmware", "6.1");
            webClient.Headers.Add("User-Agent", "Telesphoreo APT-HTTP/1.0.999");
            // Attempt to download packages file (try/catch hell)
            try
            {
                Console.WriteLine("Attempting to download " + link + "Packages.bz2");
                webClient.DownloadFile(new Uri(link + "Packages.bz2"), $"./output/{cleanLink}/Packages.bz2");
                FileStream packagesBz2 = new FileInfo($"./output/{cleanLink}/Packages.bz2").OpenRead();
                FileStream packagesBz2Decompressed = File.Create($"./output/{cleanLink}/Packages");
                BZip2.Decompress(packagesBz2, packagesBz2Decompressed, true);
            }
            catch (Exception e)
            {
                try
                {
                    Console.WriteLine("Could not download " + link + "Packages.bz2: " + e.Message);
                    Console.WriteLine("Attempting to download " + link + "Packages.gz");
                    webClient.DownloadFile(new Uri(link + "Packages.gz"), $"./output/{cleanLink}/Packages.gz");
                    FileStream packagesGz = new FileInfo($"./output/{cleanLink}/Packages.gz").OpenRead();
                    FileStream packagesGzDecompressed = File.Create($"./output/{cleanLink}/Packages");
                    GZip.Decompress(packagesGz, packagesGzDecompressed, true);
                }
                catch (Exception _e)
                {
                    try
                    {
                        Console.WriteLine("Could not download " + link + "Packages.gz: " + _e.Message);
                        Console.WriteLine("Attempting to download " + link + "Packages");
                        webClient.DownloadFile(new Uri(link + "Packages"), $"./output/{cleanLink}/Packages");
                    }
                    catch (Exception __e)
                    {
                        Console.WriteLine("Could not download " + link + "Packages: " + __e.Message);
                        Console.WriteLine("Could not locate packages file in " + link);
                        Console.WriteLine(__e.Message);
                        throw;
                    }
                }
            }
            Thread.Sleep(500);
            // Clean list of package links, names, and versions
            List<CydiaPackage> packages = new List<CydiaPackage>();
            foreach (string s in File.ReadAllText($"./output/{cleanLink}/Packages").Split("\n\n"))
            {
                string name = "";
                string version = "";
                string _link = "";
                foreach(string s2 in s.Split('\n'))
                {
                    if (s2.StartsWith("Package: "))
                    {
                        name = s2.Remove(0, 8).Trim();
                    }
                    else if (s2.StartsWith("Version: "))
                    {
                        version = s2.Remove(0, 8).Trim();
                    }
                    else if (s2.StartsWith("Filename: "))
                    {
                        _link = s2.Remove(0, 9).Trim();
                    }
                }
                packages.Add(new CydiaPackage(_link, name, version));
            }
            // remove last one because ????
            packages.RemoveAt(packages.Count - 1);
            List<string> failed = new List<string>();
            foreach(CydiaPackage p in packages)
            {
                // Download all packages on repo
                Random r = new Random();
                try
                {
                    string[] choppedUp = p.link.Split('/');
                    string fileToDownload = keepOg ? $"./output/{cleanLink}/" + choppedUp[choppedUp.Length - 1] : $"./output/{cleanLink}/" + p.name + "-" + p.version + ".deb";
                    if (File.Exists(fileToDownload))
                    {
                        fileToDownload += "_" + r.Next(0000, 9999);
                    }
                    webClient.DownloadFile(new Uri(link + p.link), fileToDownload);
                    Console.WriteLine("Successfully downloaded " + link + p.link + " as " + fileToDownload);
                }
                catch (Exception e)
                {
                    Console.WriteLine("Could not download " + link + p.link);
                    Console.WriteLine(e.Message);
                    failed.Add(link + p.link);
                }
                Thread.Sleep(options.delay);
            }
            Console.WriteLine("Finished downloading " + link);
            if(failed.Count != 0) File.WriteAllLines($"./output/{cleanLink}/failed.txt", failed);
        }

        static void DownloadInstallerRepo(string link)
        {
            if (!link.StartsWith("https://") && !link.StartsWith("http://"))
            {
                link = "http://" + link;
            }
            if (!link.EndsWith("/"))
            {
                link += "/";
            }
            string cleanLink = link.Replace("http://", "").Replace("/", "_");
            WebClient webClient = new WebClient();
            try
            {
                Console.WriteLine("Attempting to download installer repo " + link);
                webClient.DownloadFile(new Uri(link), $"./output/{cleanLink}/packages.xml");
            }
            catch (Exception e)
            {
                Console.WriteLine("Could not download package list from " + link + ": " + e.Message);
                return;
            }
            List<string> plist = new List<string>(File.ReadAllLines($"./output/{cleanLink}/packages.xml"));
            List<string> packages = new List<string>();
            int i = 1;
            foreach (string s in plist)
            {
                if (s.Contains("ocation</key>"))
                {
                    packages.Add(plist[i].Split('<')[1].Remove(0, 7));
                }
                i++;
            }
            foreach (string s in packages)
            {
                Random r = new Random();
                try
                {
                    string[] choppedUp = s.Split('/');
                    string fileToDownload = $"./output/{cleanLink}/" + choppedUp[choppedUp.Length - 1];
                    if (File.Exists(fileToDownload))
                    {
                        fileToDownload += "_" + r.Next(0000, 9999);
                    }
                    webClient.DownloadFile(new Uri(s), fileToDownload);
                    Console.WriteLine("Successfully downloaded " + s);
                }
                catch (Exception e)
                {
                    Console.WriteLine("Could not download " + s);
                    Console.WriteLine(e.Message);
                }
            }
        }
    }
}