diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml new file mode 100644 index 0000000..0b0e6ea --- /dev/null +++ b/.github/workflows/dotnet.yml @@ -0,0 +1,30 @@ +name: .NET + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Setup .NET + uses: actions/setup-dotnet@v1 + with: + dotnet-version: 3.1.x + - name: Restore dependencies + run: dotnet restore + - name: Build + run: dotnet build --no-restore + - name: Test + run: dotnet test --no-build --verbosity normal + - name: capture build artifacts + uses: actions/upload-artifact@v2 + with: + name: Build + path: RepoFullDownloader-Core/bin/Debug/netcoreapp3.1/ diff --git a/LICENSE b/LICENSE index ecc0572..e2d878c 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022 KawaiiZenbo +Copyright (c) 2021 Chloe (kawaiizenbo) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 9f0cfdd..2181e58 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# RepoFullDownloader +# RepoFullDownloader-Core Cross-Platform Cydia/Installer repo downloader ## Requirements @@ -7,4 +7,4 @@ An internet connection. ## Command Line Options `dotnet RepoFullDownloader-Core.dll ` -Running with no URL will use the repo list from `repos.txt` \ No newline at end of file +Running with no URL will use the repo list from `options.json` diff --git a/RepoFullDownloader-Core/DataClasses.cs b/RepoFullDownloader-Core/DataClasses.cs new file mode 100644 index 0000000..22086b2 --- /dev/null +++ b/RepoFullDownloader-Core/DataClasses.cs @@ -0,0 +1,62 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace RepoFullDownloader +{ + class Options + { + public bool originalFilenames { get; set; } = false; + public Repo[] repos { get; set; } = { + new Repo() + { + url = "http://repo.kawaiizenbo.me/", + type = "cydia" + }, + new Repo() + { + url = "http://apptapp.saurik.com/", + type = "installer" + }, + new Repo() + { + url = "http://apt.saurik.com/", + type = "dist", + distAttributes = new DistAttributes() + { + suites = "ios/", + components = "main", + } + } + }; + public int delay { get; set; } = 1; + } + + class Repo + { + public string url { get; set; } + public string type { get; set; } + public DistAttributes distAttributes { get; set; } = null; + } + + class DistAttributes + { + public string suites { get; set; } + public string components { get; set; } + } + + class CydiaPackage + { + public CydiaPackage(string _link, string _name, string _version) + { + this.link = _link; + this.name = _name; + this.version = _version; + } + public string link { get; set; } + public string name { get; set; } + public string version { get; set; } + } +} diff --git a/RepoFullDownloader-Core/Program.cs b/RepoFullDownloader-Core/Program.cs index 4c48f7e..e7475e9 100644 --- a/RepoFullDownloader-Core/Program.cs +++ b/RepoFullDownloader-Core/Program.cs @@ -3,21 +3,21 @@ using ICSharpCode.SharpZipLib.GZip; using PlistCS; +using RepoFullDownloader; + using System; using System.Collections.Generic; using System.IO; using System.Net; using System.Net.Http; -using System.Text.RegularExpressions; +using System.Text.Json; using System.Threading; namespace RepoFullDownloader_Core { class Program { - private static List repos = new List(); - private static bool originalFilenames; - private static int delayMS; + private static Options options = new Options(); static void Main(string[] args) { @@ -33,19 +33,17 @@ namespace RepoFullDownloader_Core { Directory.CreateDirectory("./output/"); } - // Load Options from 'options.ini' - if (!File.Exists("./options.ini")) + // Load Options from 'options.json' + if (!File.Exists("./options.json")) { - Console.WriteLine("Could not find options.ini"); + Console.WriteLine("Could not find options.json"); Console.WriteLine("Generating example..."); // generate example options - File.WriteAllText("options.ini", - "delayMS=1\n" + - "originalFilenames=false"); + File.WriteAllText("./options.json", JsonSerializer.Serialize(new Options())); + return; } - string[] options = File.ReadAllLines("options.ini"); - delayMS = int.Parse(options[0].Split('=')[1]); - originalFilenames = bool.Parse(options[1].Split('=')[1]); + string optionsJson = File.ReadAllText("./options.json"); + options = JsonSerializer.Deserialize(optionsJson); if (args.Length != 0) { string url = args[0]; @@ -69,47 +67,30 @@ namespace RepoFullDownloader_Core } else { - if (File.Exists("repos.txt")) + + foreach (Repo r in options.repos) { - foreach (string r in File.ReadAllLines("repos.txt")) + if (r.type.ToLower() == "installer") { - if (r.StartsWith('#')) continue; - string[] repoWAttributes = r.Split(' '); - try - { - switch (repoWAttributes[1]) - { - case "installer": - DownloadInstallerRepo(repoWAttributes[0]); - break; - case "cydia": - DownloadRepo(repoWAttributes[0]); - break; - case "dist": - DownloadDistRepo(repoWAttributes[0], repoWAttributes[2], repoWAttributes[3]); - break; - default: - Console.WriteLine($"Invalid repo type {repoWAttributes[1]} on {repoWAttributes[0]}"); - break; - } - } - catch (Exception e) - { - Console.WriteLine($"Invalid formatting on {r}"); - Console.WriteLine(e); - } + DownloadInstallerRepo(r.url); + } + else if (r.type.ToLower() == "cydia") + { + DownloadRepo(r.url); + } + else if (r.type.ToLower() == "dist") + { + if(r.distAttributes == null) + { + Console.WriteLine($"No dist attributes were defined for {r.url} :("); + return; + } + DownloadDistRepo(r.url, r.distAttributes); + } + else + { + Console.WriteLine($"Invalid repo type {r.type} on {r.url}"); } - } - else - { - Console.WriteLine("Could not find repos.txt"); - Console.WriteLine("Generating example..."); - // generate example repo list - File.WriteAllText("repos.txt", - "http://repo.kawaiizenbo.me cydia\n" + - "http://apptapp.saurik.com installer\n" + - "http://apt.saurik.com dist ios/ main" - ); } } @@ -149,7 +130,7 @@ namespace RepoFullDownloader_Core { Console.WriteLine("Could not download " + link + "Packages.bz2: " + e.Message); Console.WriteLine("Attempting to download " + link + "Packages.gz"); - Stream packagesGz = webClient.GetStreamAsync(link + "Packages.gz").Result; + Stream packagesGz = webClient.GetStreamAsync(link + "Packages.bz2").Result; FileStream packagesGzDecompressed = File.Create($"./output/{cleanLink}/Packages"); GZip.Decompress(packagesGz, packagesGzDecompressed, true); } @@ -208,14 +189,14 @@ namespace RepoFullDownloader_Core try { string[] choppedUp = p.link.Split('/'); - string fileToDownload = originalFilenames ? $"./output/{cleanLink}/" + choppedUp[choppedUp.Length - 1].Replace("/", "_").Replace(":", "_") : $"./output/{cleanLink}/" + p.name.Replace("/", "_").Replace(":", "_") + "-" + p.version.Replace("/", "_").Replace(":", "_") + ".deb"; + string fileToDownload = options.originalFilenames ? $"./output/{cleanLink}/" + choppedUp[choppedUp.Length - 1].Replace("/", "_").Replace(":", "_") : $"./output/{cleanLink}/" + p.name.Replace("/", "_").Replace(":", "_") + "-" + p.version.Replace("/", "_").Replace(":", "_") + ".deb"; if (File.Exists(fileToDownload)) { fileToDownload += "_" + r.Next(0000, 9999); } using (WebClient wc = new WebClient()) { - wc.DownloadFile(new Uri(link + p.link), fileToDownload); + wc.DownloadFileAsync(new Uri(link + p.link), fileToDownload); } Console.WriteLine("Successfully downloaded " + link + p.link + " as " + fileToDownload); } @@ -225,7 +206,7 @@ namespace RepoFullDownloader_Core Console.WriteLine(e.Message); failed.Add(link + p.link); } - Thread.Sleep(delayMS); + Thread.Sleep(options.delay); } Console.WriteLine("Finished downloading " + link); if(failed.Count != 0) File.WriteAllLines($"./output/{cleanLink}/failed.txt", failed); @@ -255,8 +236,7 @@ namespace RepoFullDownloader_Core Console.WriteLine("Could not download package list from " + link + ": " + e.Message); return; } - Dictionary plist = (Dictionary)Plist.readPlistSource(Regex.Replace(File.ReadAllText($"./output/{cleanLink}/packages.plist"), - "", "")); + Dictionary plist = (Dictionary)Plist.readPlist($"./output/{cleanLink}/packages.plist"); foreach (Dictionary d in (List)plist["packages"]) { Random r = new Random(); @@ -301,7 +281,7 @@ namespace RepoFullDownloader_Core } } - static void DownloadDistRepo(string link, string suites, string components) + static void DownloadDistRepo(string link, DistAttributes da) { // clean up link so no issues can ever arise if (!link.StartsWith("https://") && !link.StartsWith("http://")) @@ -315,15 +295,15 @@ namespace RepoFullDownloader_Core // make that good dist path string distPath; - if(!suites.EndsWith("/")) + if(!da.suites.EndsWith("/")) { - suites += "/"; + da.suites += "/"; } - if (!components.EndsWith("/")) + if (!da.components.EndsWith("/")) { - components += "/"; + da.components += "/"; } - distPath = suites + components; + distPath = da.suites + da.components; string poolpfLink = link + "dists/" + distPath + "binary-iphoneos-arm/"; string cleanLink = link.TrimEnd('/').Replace("http://", "").Replace("https://", "").Replace("/", "_").Replace(":", "_"); @@ -407,7 +387,7 @@ namespace RepoFullDownloader_Core try { string[] choppedUp = p.link.Split('/'); - string fileToDownload = originalFilenames ? $"./output/{cleanLink}-({distPath.Replace("/", "_").Replace(":", "_")})/" + choppedUp[choppedUp.Length - 1].Replace("/", "_").Replace(":", "_") : $"./output/{cleanLink}-({distPath.Replace("/", "_").Replace(":", "_")})/" + p.name.Replace("/", "_").Replace(":", "_") + "-" + p.version.Replace("/", "_").Replace(":", "_") + ".deb"; + string fileToDownload = options.originalFilenames ? $"./output/{cleanLink}-({distPath.Replace("/", "_").Replace(":", "_")})/" + choppedUp[choppedUp.Length - 1].Replace("/", "_").Replace(":", "_") : $"./output/{cleanLink}-({distPath.Replace("/", "_").Replace(":", "_")})/" + p.name.Replace("/", "_").Replace(":", "_") + "-" + p.version.Replace("/", "_").Replace(":", "_") + ".deb"; if (File.Exists(fileToDownload)) { fileToDownload += "_" + r.Next(0000, 9999); @@ -424,23 +404,10 @@ namespace RepoFullDownloader_Core Console.WriteLine(e.Message); failed.Add(link + p.link); } - Thread.Sleep(delayMS); + Thread.Sleep(options.delay); } Console.WriteLine("Finished downloading " + link); if (failed.Count != 0) File.WriteAllLines($"./output/{cleanLink}-({distPath.Replace("/", "_").Replace(":", "_")})/failed.txt", failed); } } - - class CydiaPackage - { - public CydiaPackage(string _link, string _name, string _version) - { - this.link = _link; - this.name = _name; - this.version = _version; - } - public string link { get; set; } - public string name { get; set; } - public string version { get; set; } - } }