dist repo support + delay + instalelr repos
This commit is contained in:
kawaiizenbo
parent
ef4b7af9e0
commit
87a75e6844
5 changed files with 1231 additions and 52 deletions
|
|
@ -1,12 +1,15 @@
|
|||
using ICSharpCode.SharpZipLib.BZip2;
|
||||
using ICSharpCode.SharpZipLib.GZip;
|
||||
|
||||
using PlistCS;
|
||||
|
||||
using RepoFullDownloader;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
|
||||
|
|
@ -20,12 +23,25 @@ namespace RepoFullDownloader_Core
|
|||
{
|
||||
Console.WriteLine("RepoFullDownloader by KawaiiZenbo");
|
||||
|
||||
ServicePointManager.ServerCertificateValidationCallback += (sender, certificate, chain, errors) =>
|
||||
{
|
||||
// security
|
||||
return true;
|
||||
};
|
||||
// Initial Checks
|
||||
if (!Directory.Exists("./output/"))
|
||||
{
|
||||
Directory.CreateDirectory("./output/");
|
||||
}
|
||||
// Load Options from 'options.json'
|
||||
if (!File.Exists("./options.json"))
|
||||
{
|
||||
Console.WriteLine("Could not find options.json");
|
||||
Console.WriteLine("Generating example...");
|
||||
// generate example options
|
||||
File.WriteAllText("./options.json", JsonSerializer.Serialize(new Options()));
|
||||
return;
|
||||
}
|
||||
string optionsJson = File.ReadAllText("./options.json");
|
||||
options = JsonSerializer.Deserialize<Options>(optionsJson);
|
||||
if (args.Length != 0)
|
||||
|
|
@ -41,7 +57,7 @@ namespace RepoFullDownloader_Core
|
|||
}
|
||||
try
|
||||
{
|
||||
DownloadRepo(url, options.originalFilenames);
|
||||
DownloadRepo(url);
|
||||
}
|
||||
catch(Exception)
|
||||
{
|
||||
|
|
@ -51,24 +67,29 @@ namespace RepoFullDownloader_Core
|
|||
}
|
||||
else
|
||||
{
|
||||
if (!File.Exists("./options.json"))
|
||||
{
|
||||
Console.WriteLine("Could not find options.json");
|
||||
Console.WriteLine("Generating example...");
|
||||
// generate example options
|
||||
File.WriteAllText("./options.json", JsonSerializer.Serialize(new Options()));
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (Repo r in options.repos)
|
||||
{
|
||||
if (r.isInstaller)
|
||||
if (r.type.ToLower() == "installer")
|
||||
{
|
||||
DownloadInstallerRepo(r.url);
|
||||
}
|
||||
else if (r.type.ToLower() == "cydia")
|
||||
{
|
||||
DownloadRepo(r.url);
|
||||
}
|
||||
else if (r.type.ToLower() == "dist")
|
||||
{
|
||||
if(r.distAttributes == null)
|
||||
{
|
||||
Console.WriteLine($"No dist attributes were defined for {r.url} :(");
|
||||
return;
|
||||
}
|
||||
DownloadDistRepo(r.url, r.distAttributes);
|
||||
}
|
||||
else
|
||||
{
|
||||
DownloadRepo(r.url, options.originalFilenames);
|
||||
Console.WriteLine($"Invalid repo type {r.type} on {r.url}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -77,7 +98,7 @@ namespace RepoFullDownloader_Core
|
|||
Console.WriteLine("done :)");
|
||||
}
|
||||
|
||||
static void DownloadRepo(string link, bool keepOg)
|
||||
static void DownloadRepo(string link)
|
||||
{
|
||||
if (!link.StartsWith("https://") && !link.StartsWith("http://"))
|
||||
{
|
||||
|
|
@ -89,18 +110,17 @@ namespace RepoFullDownloader_Core
|
|||
}
|
||||
string cleanLink = link.TrimEnd('/').Replace("http://", "").Replace("https://", "").Replace("/", "_").Replace(":", "_");
|
||||
Directory.CreateDirectory($"./output/{cleanLink}");
|
||||
WebClient webClient = new WebClient();
|
||||
HttpClient webClient = new HttpClient();
|
||||
// headers because some repos are 'interesting'
|
||||
webClient.Headers.Add("X-Machine", "iPod4,1");
|
||||
webClient.Headers.Add("X-Unique-ID", "0000000000000000000000000000000000000000");
|
||||
webClient.Headers.Add("X-Firmware", "6.1");
|
||||
webClient.Headers.Add("User-Agent", "Telesphoreo APT-HTTP/1.0.999");
|
||||
webClient.DefaultRequestHeaders.Add("X-Machine", "iPod4,1");
|
||||
webClient.DefaultRequestHeaders.Add("X-Unique-ID", "0000000000000000000000000000000000000000");
|
||||
webClient.DefaultRequestHeaders.Add("X-Firmware", "6.1");
|
||||
webClient.DefaultRequestHeaders.Add("User-Agent", "Telesphoreo APT-HTTP/1.0.999");
|
||||
// Attempt to download packages file (try/catch hell)
|
||||
try
|
||||
{
|
||||
Console.WriteLine("Attempting to download " + link + "Packages.bz2");
|
||||
webClient.DownloadFile(new Uri(link + "Packages.bz2"), $"./output/{cleanLink}/Packages.bz2");
|
||||
FileStream packagesBz2 = new FileInfo($"./output/{cleanLink}/Packages.bz2").OpenRead();
|
||||
Stream packagesBz2 = webClient.GetStreamAsync(link + "Packages.bz2").Result;
|
||||
FileStream packagesBz2Decompressed = File.Create($"./output/{cleanLink}/Packages");
|
||||
BZip2.Decompress(packagesBz2, packagesBz2Decompressed, true);
|
||||
}
|
||||
|
|
@ -110,8 +130,7 @@ namespace RepoFullDownloader_Core
|
|||
{
|
||||
Console.WriteLine("Could not download " + link + "Packages.bz2: " + e.Message);
|
||||
Console.WriteLine("Attempting to download " + link + "Packages.gz");
|
||||
webClient.DownloadFile(new Uri(link + "Packages.gz"), $"./output/{cleanLink}/Packages.gz");
|
||||
FileStream packagesGz = new FileInfo($"./output/{cleanLink}/Packages.gz").OpenRead();
|
||||
Stream packagesGz = webClient.GetStreamAsync(link + "Packages.bz2").Result;
|
||||
FileStream packagesGzDecompressed = File.Create($"./output/{cleanLink}/Packages");
|
||||
GZip.Decompress(packagesGz, packagesGzDecompressed, true);
|
||||
}
|
||||
|
|
@ -121,7 +140,10 @@ namespace RepoFullDownloader_Core
|
|||
{
|
||||
Console.WriteLine("Could not download " + link + "Packages.gz: " + _e.Message);
|
||||
Console.WriteLine("Attempting to download " + link + "Packages");
|
||||
webClient.DownloadFile(new Uri(link + "Packages"), $"./output/{cleanLink}/Packages");
|
||||
using (StreamWriter outputFile = new StreamWriter($"./output/{cleanLink}/Packages"))
|
||||
{
|
||||
outputFile.WriteLine(webClient.GetStreamAsync(link + "Packages").Result);
|
||||
}
|
||||
}
|
||||
catch (Exception __e)
|
||||
{
|
||||
|
|
@ -167,12 +189,15 @@ namespace RepoFullDownloader_Core
|
|||
try
|
||||
{
|
||||
string[] choppedUp = p.link.Split('/');
|
||||
string fileToDownload = keepOg ? $"./output/{cleanLink}/" + choppedUp[choppedUp.Length - 1] : $"./output/{cleanLink}/" + p.name + "-" + p.version + ".deb";
|
||||
string fileToDownload = options.originalFilenames ? $"./output/{cleanLink}/" + choppedUp[choppedUp.Length - 1].Replace("/", "_").Replace(":", "_") : $"./output/{cleanLink}/" + p.name.Replace("/", "_").Replace(":", "_") + "-" + p.version.Replace("/", "_").Replace(":", "_") + ".deb";
|
||||
if (File.Exists(fileToDownload))
|
||||
{
|
||||
fileToDownload += "_" + r.Next(0000, 9999);
|
||||
}
|
||||
webClient.DownloadFile(new Uri(link + p.link), fileToDownload);
|
||||
using (WebClient wc = new WebClient())
|
||||
{
|
||||
wc.DownloadFileAsync(new Uri(link + p.link), fileToDownload);
|
||||
}
|
||||
Console.WriteLine("Successfully downloaded " + link + p.link + " as " + fileToDownload);
|
||||
}
|
||||
catch (Exception e)
|
||||
|
|
@ -197,49 +222,192 @@ namespace RepoFullDownloader_Core
|
|||
{
|
||||
link += "/";
|
||||
}
|
||||
string cleanLink = link.Replace("http://", "").Replace("/", "_");
|
||||
string cleanLink = link.TrimEnd('/').Replace("http://", "").Replace("https://", "").Replace("/", "_").Replace(":", "_");
|
||||
Directory.CreateDirectory($"./output/{cleanLink}");
|
||||
WebClient webClient = new WebClient();
|
||||
webClient.Headers.Add("User-Agent", "AppTapp Installer/3.0 (iPhone/1.1, like CFNetwork/100.0)");
|
||||
try
|
||||
{
|
||||
Console.WriteLine("Attempting to download installer repo " + link);
|
||||
webClient.DownloadFile(new Uri(link), $"./output/{cleanLink}/packages.xml");
|
||||
webClient.DownloadFile(link, $"./output/{cleanLink}/packages.plist");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine("Could not download package list from " + link + ": " + e.Message);
|
||||
return;
|
||||
}
|
||||
List<string> plist = new List<string>(File.ReadAllLines($"./output/{cleanLink}/packages.xml"));
|
||||
List<string> packages = new List<string>();
|
||||
int i = 1;
|
||||
foreach (string s in plist)
|
||||
{
|
||||
if (s.Contains("ocation</key>"))
|
||||
{
|
||||
packages.Add(plist[i].Split('<')[1].Remove(0, 7));
|
||||
}
|
||||
i++;
|
||||
}
|
||||
foreach (string s in packages)
|
||||
Dictionary<string, object> plist = (Dictionary<string, object>)Plist.readPlist($"./output/{cleanLink}/packages.plist");
|
||||
foreach (Dictionary<string, object> d in (List<object>)plist["packages"])
|
||||
{
|
||||
Random r = new Random();
|
||||
try
|
||||
{
|
||||
string[] choppedUp = s.Split('/');
|
||||
string[] choppedUp = d["location"].ToString().Split('/');
|
||||
string fileToDownload = $"./output/{cleanLink}/" + choppedUp[choppedUp.Length - 1];
|
||||
if (File.Exists(fileToDownload))
|
||||
{
|
||||
fileToDownload += "_" + r.Next(0000, 9999);
|
||||
}
|
||||
webClient.DownloadFile(new Uri(s), fileToDownload);
|
||||
Console.WriteLine("Successfully downloaded " + s);
|
||||
webClient.DownloadFile(new Uri((string)d["location"]), fileToDownload);
|
||||
Console.WriteLine("Successfully downloaded " + (string)d["location"]);
|
||||
}
|
||||
catch (KeyNotFoundException)
|
||||
{
|
||||
string[] choppedUp = d["url"].ToString().Split('/');
|
||||
string fileToDownload = $"./output/{cleanLink}/" + choppedUp[choppedUp.Length - 1];
|
||||
if (File.Exists(fileToDownload))
|
||||
{
|
||||
fileToDownload += "_" + r.Next(0000, 9999);
|
||||
}
|
||||
webClient.DownloadFile(new Uri((string)d["url"]), fileToDownload);
|
||||
Console.WriteLine("Successfully downloaded " + (string)d["url"]);
|
||||
Dictionary<string, object> pl = (Dictionary<string, object>)Plist.readPlist(fileToDownload);
|
||||
{
|
||||
string[] choppedUp2 = pl["location"].ToString().Split('/');
|
||||
string fileToDownload2 = $"./output/{cleanLink}/" + choppedUp2[choppedUp2.Length - 1];
|
||||
if (File.Exists(fileToDownload2))
|
||||
{
|
||||
fileToDownload2 += "_" + r.Next(0000, 9999);
|
||||
}
|
||||
webClient.DownloadFile(new Uri((string)pl["location"]), fileToDownload2);
|
||||
Console.WriteLine("Successfully downloaded " + (string)pl["location"]);
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine("Could not download " + s);
|
||||
Console.WriteLine("Could not download " + (string)d["location"]);
|
||||
Console.WriteLine(e.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void DownloadDistRepo(string link, DistAttributes da)
|
||||
{
|
||||
// clean up link so no issues can ever arise
|
||||
if (!link.StartsWith("https://") && !link.StartsWith("http://"))
|
||||
{
|
||||
link = "http://" + link;
|
||||
}
|
||||
if (!link.EndsWith("/"))
|
||||
{
|
||||
link += "/";
|
||||
}
|
||||
|
||||
// make that good dist path
|
||||
string distPath;
|
||||
if(!da.suites.EndsWith("/"))
|
||||
{
|
||||
da.suites += "/";
|
||||
}
|
||||
if (!da.components.EndsWith("/"))
|
||||
{
|
||||
da.components += "/";
|
||||
}
|
||||
distPath = da.suites + da.components;
|
||||
string poolpfLink = link + "dists/" + distPath + "binary-iphoneos-arm/";
|
||||
|
||||
string cleanLink = link.TrimEnd('/').Replace("http://", "").Replace("https://", "").Replace("/", "_").Replace(":", "_");
|
||||
Directory.CreateDirectory($"./output/{cleanLink}-({distPath.Replace("/", "_").Replace(":", "_")})");
|
||||
HttpClient webClient = new HttpClient();
|
||||
// headers because some repos are 'interesting'
|
||||
webClient.DefaultRequestHeaders.Add("X-Machine", "iPod4,1");
|
||||
webClient.DefaultRequestHeaders.Add("X-Unique-ID", "0000000000000000000000000000000000000000");
|
||||
webClient.DefaultRequestHeaders.Add("X-Firmware", "6.1");
|
||||
webClient.DefaultRequestHeaders.Add("User-Agent", "Telesphoreo APT-HTTP/1.0.999");
|
||||
// Attempt to download packages file (try/catch hell)
|
||||
try
|
||||
{
|
||||
Console.WriteLine("Attempting to download " + poolpfLink + "Packages.bz2");
|
||||
Stream packagesBz2 = webClient.GetStreamAsync(poolpfLink + "Packages.bz2").Result;
|
||||
FileStream packagesBz2Decompressed = File.Create($"./output/{cleanLink}-({distPath.Replace("/", "_").Replace(":", "_")})/Packages");
|
||||
BZip2.Decompress(packagesBz2, packagesBz2Decompressed, true);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
try
|
||||
{
|
||||
Console.WriteLine("Could not download " + poolpfLink + "Packages.bz2: " + e.Message);
|
||||
Console.WriteLine("Attempting to download " + poolpfLink + "Packages.gz");
|
||||
Stream packagesGz = webClient.GetStreamAsync(poolpfLink + "Packages.gz").Result;
|
||||
FileStream packagesGzDecompressed = File.Create($"./output/{cleanLink}-({distPath.Replace("/", "_").Replace(":", "_")})/Packages");
|
||||
GZip.Decompress(packagesGz, packagesGzDecompressed, true);
|
||||
}
|
||||
catch (Exception _e)
|
||||
{
|
||||
try
|
||||
{
|
||||
Console.WriteLine("Could not download " + poolpfLink + "Packages.gz: " + _e.Message);
|
||||
Console.WriteLine("Attempting to download " + poolpfLink + "Packages");
|
||||
using (StreamWriter outputFile = new StreamWriter($"./output/{cleanLink}-({distPath.Replace("/", "_").Replace(":", "_")})/Packages"))
|
||||
{
|
||||
outputFile.WriteLine(webClient.GetStreamAsync(poolpfLink + "Packages").Result);
|
||||
}
|
||||
}
|
||||
catch (Exception __e)
|
||||
{
|
||||
Console.WriteLine("Could not download " + poolpfLink + "Packages: " + __e.Message);
|
||||
Console.WriteLine("Could not locate packages file in " + poolpfLink);
|
||||
Console.WriteLine(__e.Message);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
Thread.Sleep(500);
|
||||
// Clean list of package links, names, and versions
|
||||
List<CydiaPackage> packages = new List<CydiaPackage>();
|
||||
foreach (string s in File.ReadAllText($"./output/{cleanLink}-({distPath.Replace("/", "_").Replace(":", "_")})/Packages").Split("\n\n"))
|
||||
{
|
||||
string name = "";
|
||||
string version = "";
|
||||
string _link = "";
|
||||
foreach (string s2 in s.Split('\n'))
|
||||
{
|
||||
if (s2.StartsWith("Package: "))
|
||||
{
|
||||
name = s2.Remove(0, 8).Trim();
|
||||
}
|
||||
else if (s2.StartsWith("Version: "))
|
||||
{
|
||||
version = s2.Remove(0, 8).Trim();
|
||||
}
|
||||
else if (s2.StartsWith("Filename: "))
|
||||
{
|
||||
_link = s2.Remove(0, 9).Trim();
|
||||
}
|
||||
}
|
||||
packages.Add(new CydiaPackage(_link, name, version));
|
||||
}
|
||||
// remove last one because ????
|
||||
packages.RemoveAt(packages.Count - 1);
|
||||
List<string> failed = new List<string>();
|
||||
foreach (CydiaPackage p in packages)
|
||||
{
|
||||
// Download all packages on repo
|
||||
Random r = new Random();
|
||||
try
|
||||
{
|
||||
string[] choppedUp = p.link.Split('/');
|
||||
string fileToDownload = options.originalFilenames ? $"./output/{cleanLink}-({distPath.Replace("/", "_").Replace(":", "_")})/" + choppedUp[choppedUp.Length - 1].Replace("/", "_").Replace(":", "_") : $"./output/{cleanLink}-({distPath.Replace("/", "_").Replace(":", "_")})/" + p.name.Replace("/", "_").Replace(":", "_") + "-" + p.version.Replace("/", "_").Replace(":", "_") + ".deb";
|
||||
if (File.Exists(fileToDownload))
|
||||
{
|
||||
fileToDownload += "_" + r.Next(0000, 9999);
|
||||
}
|
||||
using (WebClient wc = new WebClient())
|
||||
{
|
||||
wc.DownloadFileAsync(new Uri(link + p.link), fileToDownload);
|
||||
}
|
||||
Console.WriteLine("Successfully downloaded " + link + p.link + " as " + fileToDownload);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine("Could not download " + link + p.link);
|
||||
Console.WriteLine(e.Message);
|
||||
failed.Add(link + p.link);
|
||||
}
|
||||
Thread.Sleep(options.delay);
|
||||
}
|
||||
Console.WriteLine("Finished downloading " + link);
|
||||
if (failed.Count != 0) File.WriteAllLines($"./output/{cleanLink}-({distPath.Replace("/", "_").Replace(":", "_")})/failed.txt", failed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue