From 87a75e6844b0fbda58342b9dcfbaa7fde184ba52 Mon Sep 17 00:00:00 2001 From: kawaiizenbo <48113593+kawaiizenbo@users.noreply.github.com> Date: Fri, 10 Dec 2021 09:23:03 -0700 Subject: [PATCH] dist repo support + delay + instalelr repos --- RepoFullDownloader-Core/App.config | 10 + RepoFullDownloader-Core/DataClasses.cs | 32 +- RepoFullDownloader-Core/Plist.cs | 961 +++++++++++++++++++++++++ RepoFullDownloader-Core/Program.cs | 252 +++++-- example-options.json | 28 +- 5 files changed, 1231 insertions(+), 52 deletions(-) create mode 100644 RepoFullDownloader-Core/App.config create mode 100644 RepoFullDownloader-Core/Plist.cs diff --git a/RepoFullDownloader-Core/App.config b/RepoFullDownloader-Core/App.config new file mode 100644 index 0000000..082a4aa --- /dev/null +++ b/RepoFullDownloader-Core/App.config @@ -0,0 +1,10 @@ + + + + + + + + \ No newline at end of file diff --git a/RepoFullDownloader-Core/DataClasses.cs b/RepoFullDownloader-Core/DataClasses.cs index 21542a2..22086b2 100644 --- a/RepoFullDownloader-Core/DataClasses.cs +++ b/RepoFullDownloader-Core/DataClasses.cs @@ -9,14 +9,42 @@ namespace RepoFullDownloader class Options { public bool originalFilenames { get; set; } = false; - public Repo[] repos { get; set; } = { new Repo() { url = "http://cydia.invoxiplaygames.uk/" , isInstaller = false}, new Repo() { url = "http://apptapp.saurik.com/", isInstaller = true } }; + public Repo[] repos { get; set; } = { + new Repo() + { + url = "http://repo.kawaiizenbo.me/", + type = "cydia" + }, + new Repo() + { + url = "http://apptapp.saurik.com/", + type = "installer" + }, + new Repo() + { + url = "http://apt.saurik.com/", + type = "dist", + distAttributes = new DistAttributes() + { + suites = "ios/", + components = "main", + } + } + }; public int delay { get; set; } = 1; } class Repo { public string url { get; set; } - public bool isInstaller { get; set; } + public string type { get; set; } + public DistAttributes distAttributes { get; set; } = null; + } + + class DistAttributes + { + public string suites { get; set; } + public string components { get; set; } } class CydiaPackage diff --git a/RepoFullDownloader-Core/Plist.cs b/RepoFullDownloader-Core/Plist.cs new file mode 100644 index 0000000..f52c364 --- /dev/null +++ b/RepoFullDownloader-Core/Plist.cs @@ -0,0 +1,961 @@ +// +// PlistCS Property List (plist) serialization and parsing library. +// +// https://github.com/animetrics/PlistCS +// +// Copyright (c) 2011 Animetrics Inc. (marc@animetrics.com) +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + + +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Xml; + +namespace PlistCS +{ + public static class Plist + { + private static List offsetTable = new List(); + private static List objectTable = new List(); + private static int refCount; + private static int objRefSize; + private static int offsetByteSize; + private static long offsetTableOffset; + + #region Public Functions + + public static object readPlist(string path) + { + using (FileStream f = new FileStream(path, FileMode.Open, FileAccess.Read)) + { + return readPlist(f, plistType.Auto); + } + } + + public static object readPlistSource(string source) + { + return readPlist(System.Text.Encoding.UTF8.GetBytes(source)); + } + + public static object readPlist(byte[] data) + { + return readPlist(new MemoryStream(data), plistType.Auto); + } + + public static plistType getPlistType(Stream stream) + { + byte[] magicHeader = new byte[8]; + stream.Read(magicHeader, 0, 8); + + if (BitConverter.ToInt64(magicHeader, 0) == 3472403351741427810) + { + return plistType.Binary; + } + else + { + return plistType.Xml; + } + } + + public static object readPlist(Stream stream, plistType type) + { + if (type == plistType.Auto) + { + type = getPlistType(stream); + stream.Seek(0, SeekOrigin.Begin); + } + + if (type == plistType.Binary) + { + using (BinaryReader reader = new BinaryReader(stream)) + { + byte[] data = reader.ReadBytes((int)reader.BaseStream.Length); + return readBinary(data); + } + } + else + { + XmlDocument xml = new XmlDocument(); + xml.XmlResolver = null; + xml.Load(stream); + return readXml(xml); + } + } + + public static void writeXml(object value, string path) + { + using (StreamWriter writer = new StreamWriter(path)) + { + writer.Write(writeXml(value)); + } + } + + public static void writeXml(object value, Stream stream) + { + using (StreamWriter writer = new StreamWriter(stream)) + { + writer.Write(writeXml(value)); + } + } + + public static string writeXml(object value) + { + using (MemoryStream ms = new MemoryStream()) + { + XmlWriterSettings xmlWriterSettings = new XmlWriterSettings(); + xmlWriterSettings.Encoding = new System.Text.UTF8Encoding(false); + xmlWriterSettings.ConformanceLevel = ConformanceLevel.Document; + xmlWriterSettings.Indent = true; + + using (XmlWriter xmlWriter = XmlWriter.Create(ms, xmlWriterSettings)) + { + xmlWriter.WriteStartDocument(); + //xmlWriter.WriteComment("DOCTYPE plist PUBLIC \"-//Apple//DTD PLIST 1.0//EN\" " + "\"http://www.apple.com/DTDs/PropertyList-1.0.dtd\""); + xmlWriter.WriteDocType("plist", "-//Apple Computer//DTD PLIST 1.0//EN", "http://www.apple.com/DTDs/PropertyList-1.0.dtd", null); + xmlWriter.WriteStartElement("plist"); + xmlWriter.WriteAttributeString("version", "1.0"); + compose(value, xmlWriter); + xmlWriter.WriteEndElement(); + xmlWriter.WriteEndDocument(); + xmlWriter.Flush(); + xmlWriter.Close(); + return System.Text.Encoding.UTF8.GetString(ms.ToArray()); + } + } + } + + public static void writeBinary(object value, string path) + { + using (BinaryWriter writer = new BinaryWriter(new FileStream(path, FileMode.Create))) + { + writer.Write(writeBinary(value)); + } + } + + public static void writeBinary(object value, Stream stream) + { + using (BinaryWriter writer = new BinaryWriter(stream)) + { + writer.Write(writeBinary(value)); + } + } + + public static byte[] writeBinary(object value) + { + offsetTable.Clear(); + objectTable.Clear(); + refCount = 0; + objRefSize = 0; + offsetByteSize = 0; + offsetTableOffset = 0; + + //Do not count the root node, subtract by 1 + int totalRefs = countObject(value) - 1; + + refCount = totalRefs; + + objRefSize = RegulateNullBytes(BitConverter.GetBytes(refCount)).Length; + + composeBinary(value); + + writeBinaryString("bplist00", false); + + offsetTableOffset = (long)objectTable.Count; + + offsetTable.Add(objectTable.Count - 8); + + offsetByteSize = RegulateNullBytes(BitConverter.GetBytes(offsetTable[offsetTable.Count - 1])).Length; + + List offsetBytes = new List(); + + offsetTable.Reverse(); + + for (int i = 0; i < offsetTable.Count; i++) + { + offsetTable[i] = objectTable.Count - offsetTable[i]; + byte[] buffer = RegulateNullBytes(BitConverter.GetBytes(offsetTable[i]), offsetByteSize); + Array.Reverse(buffer); + offsetBytes.AddRange(buffer); + } + + objectTable.AddRange(offsetBytes); + + objectTable.AddRange(new byte[6]); + objectTable.Add(Convert.ToByte(offsetByteSize)); + objectTable.Add(Convert.ToByte(objRefSize)); + + var a = BitConverter.GetBytes((long)totalRefs + 1); + Array.Reverse(a); + objectTable.AddRange(a); + + objectTable.AddRange(BitConverter.GetBytes((long)0)); + a = BitConverter.GetBytes(offsetTableOffset); + Array.Reverse(a); + objectTable.AddRange(a); + + return objectTable.ToArray(); + } + + #endregion + + #region Private Functions + + private static object readXml(XmlDocument xml) + { + XmlNode rootNode = xml.DocumentElement.ChildNodes[0]; + return parse(rootNode); + } + + private static object readBinary(byte[] data) + { + offsetTable.Clear(); + List offsetTableBytes = new List(); + objectTable.Clear(); + refCount = 0; + objRefSize = 0; + offsetByteSize = 0; + offsetTableOffset = 0; + + List bList = new List(data); + + List trailer = bList.GetRange(bList.Count - 32, 32); + + parseTrailer(trailer); + + objectTable = bList.GetRange(0, (int)offsetTableOffset); + + offsetTableBytes = bList.GetRange((int)offsetTableOffset, bList.Count - (int)offsetTableOffset - 32); + + parseOffsetTable(offsetTableBytes); + + return parseBinary(0); + } + + private static Dictionary parseDictionary(XmlNode node) + { + XmlNodeList children = node.ChildNodes; + if (children.Count % 2 != 0) + { + throw new DataMisalignedException("Dictionary elements must have an even number of child nodes"); + } + + Dictionary dict = new Dictionary(); + + for (int i = 0; i < children.Count; i += 2) + { + XmlNode keynode = children[i]; + XmlNode valnode = children[i + 1]; + + if (keynode.Name != "key") + { + throw new ApplicationException("expected a key node"); + } + + object result = parse(valnode); + + if (result != null) + { + dict.Add(keynode.InnerText, result); + } + } + + return dict; + } + + private static List parseArray(XmlNode node) + { + List array = new List(); + + foreach (XmlNode child in node.ChildNodes) + { + object result = parse(child); + if (result != null) + { + array.Add(result); + } + } + + return array; + } + + private static void composeArray(List value, XmlWriter writer) + { + writer.WriteStartElement("array"); + foreach (object obj in value) + { + compose(obj, writer); + } + writer.WriteEndElement(); + } + + private static object parse(XmlNode node) + { + switch (node.Name) + { + case "dict": + return parseDictionary(node); + case "array": + return parseArray(node); + case "string": + return node.InnerText; + case "integer": + // int result; + //int.TryParse(node.InnerText, System.Globalization.NumberFormatInfo.InvariantInfo, out result); + return Convert.ToInt32(node.InnerText, System.Globalization.NumberFormatInfo.InvariantInfo); + case "real": + return Convert.ToDouble(node.InnerText, System.Globalization.NumberFormatInfo.InvariantInfo); + case "false": + return false; + case "true": + return true; + case "null": + return null; + case "date": + return XmlConvert.ToDateTime(node.InnerText, XmlDateTimeSerializationMode.Utc); + case "data": + return Convert.FromBase64String(node.InnerText); + } + + throw new ApplicationException(String.Format("Plist Node `{0}' is not supported", node.Name)); + } + + private static void compose(object value, XmlWriter writer) + { + + if (value == null || value is string) + { + writer.WriteElementString("string", value as string); + } + else if (value is int || value is long) + { + writer.WriteElementString("integer", ((int)value).ToString(System.Globalization.NumberFormatInfo.InvariantInfo)); + } + else if (value is System.Collections.Generic.Dictionary || + value.GetType().ToString().StartsWith("System.Collections.Generic.Dictionary`2[System.String")) + { + //Convert to Dictionary + Dictionary dic = value as Dictionary; + if (dic == null) + { + dic = new Dictionary(); + IDictionary idic = (IDictionary)value; + foreach (var key in idic.Keys) + { + dic.Add(key.ToString(), idic[key]); + } + } + writeDictionaryValues(dic, writer); + } + else if (value is List) + { + composeArray((List)value, writer); + } + else if (value is byte[]) + { + writer.WriteElementString("data", Convert.ToBase64String((Byte[])value)); + } + else if (value is float || value is double) + { + writer.WriteElementString("real", ((double)value).ToString(System.Globalization.NumberFormatInfo.InvariantInfo)); + } + else if (value is DateTime) + { + DateTime time = (DateTime)value; + string theString = XmlConvert.ToString(time, XmlDateTimeSerializationMode.Utc); + writer.WriteElementString("date", theString);//, "yyyy-MM-ddTHH:mm:ssZ")); + } + else if (value is bool) + { + writer.WriteElementString(value.ToString().ToLower(), ""); + } + else + { + throw new Exception(String.Format("Value type '{0}' is unhandled", value.GetType().ToString())); + } + } + + private static void writeDictionaryValues(Dictionary dictionary, XmlWriter writer) + { + writer.WriteStartElement("dict"); + foreach (string key in dictionary.Keys) + { + object value = dictionary[key]; + writer.WriteElementString("key", key); + compose(value, writer); + } + writer.WriteEndElement(); + } + + private static int countObject(object value) + { + int count = 0; + switch (value.GetType().ToString()) + { + case "System.Collections.Generic.Dictionary`2[System.String,System.Object]": + Dictionary dict = (Dictionary)value; + foreach (string key in dict.Keys) + { + count += countObject(dict[key]); + } + count += dict.Keys.Count; + count++; + break; + case "System.Collections.Generic.List`1[System.Object]": + List list = (List)value; + foreach (object obj in list) + { + count += countObject(obj); + } + count++; + break; + default: + count++; + break; + } + return count; + } + + private static byte[] writeBinaryDictionary(Dictionary dictionary) + { + List buffer = new List(); + List header = new List(); + List refs = new List(); + for (int i = dictionary.Count - 1; i >= 0; i--) + { + var o = new object[dictionary.Count]; + dictionary.Values.CopyTo(o, 0); + composeBinary(o[i]); + offsetTable.Add(objectTable.Count); + refs.Add(refCount); + refCount--; + } + for (int i = dictionary.Count - 1; i >= 0; i--) + { + var o = new string[dictionary.Count]; + dictionary.Keys.CopyTo(o, 0); + composeBinary(o[i]);//); + offsetTable.Add(objectTable.Count); + refs.Add(refCount); + refCount--; + } + + if (dictionary.Count < 15) + { + header.Add(Convert.ToByte(0xD0 | Convert.ToByte(dictionary.Count))); + } + else + { + header.Add(0xD0 | 0xf); + header.AddRange(writeBinaryInteger(dictionary.Count, false)); + } + + + foreach (int val in refs) + { + byte[] refBuffer = RegulateNullBytes(BitConverter.GetBytes(val), objRefSize); + Array.Reverse(refBuffer); + buffer.InsertRange(0, refBuffer); + } + + buffer.InsertRange(0, header); + + + objectTable.InsertRange(0, buffer); + + return buffer.ToArray(); + } + + private static byte[] composeBinaryArray(List objects) + { + List buffer = new List(); + List header = new List(); + List refs = new List(); + + for (int i = objects.Count - 1; i >= 0; i--) + { + composeBinary(objects[i]); + offsetTable.Add(objectTable.Count); + refs.Add(refCount); + refCount--; + } + + if (objects.Count < 15) + { + header.Add(Convert.ToByte(0xA0 | Convert.ToByte(objects.Count))); + } + else + { + header.Add(0xA0 | 0xf); + header.AddRange(writeBinaryInteger(objects.Count, false)); + } + + foreach (int val in refs) + { + byte[] refBuffer = RegulateNullBytes(BitConverter.GetBytes(val), objRefSize); + Array.Reverse(refBuffer); + buffer.InsertRange(0, refBuffer); + } + + buffer.InsertRange(0, header); + + objectTable.InsertRange(0, buffer); + + return buffer.ToArray(); + } + + private static byte[] composeBinary(object obj) + { + byte[] value; + switch (obj.GetType().ToString()) + { + case "System.Collections.Generic.Dictionary`2[System.String,System.Object]": + value = writeBinaryDictionary((Dictionary)obj); + return value; + + case "System.Collections.Generic.List`1[System.Object]": + value = composeBinaryArray((List)obj); + return value; + + case "System.Byte[]": + value = writeBinaryByteArray((byte[])obj); + return value; + + case "System.Double": + value = writeBinaryDouble((double)obj); + return value; + + case "System.Int32": + value = writeBinaryInteger((int)obj, true); + return value; + + case "System.String": + value = writeBinaryString((string)obj, true); + return value; + + case "System.DateTime": + value = writeBinaryDate((DateTime)obj); + return value; + + case "System.Boolean": + value = writeBinaryBool((bool)obj); + return value; + + default: + return new byte[0]; + } + } + + public static byte[] writeBinaryDate(DateTime obj) + { + List buffer = new List(RegulateNullBytes(BitConverter.GetBytes(PlistDateConverter.ConvertToAppleTimeStamp(obj)), 8)); + buffer.Reverse(); + buffer.Insert(0, 0x33); + objectTable.InsertRange(0, buffer); + return buffer.ToArray(); + } + + public static byte[] writeBinaryBool(bool obj) + { + List buffer = new List(new byte[1] { (bool)obj ? (byte)9 : (byte)8 }); + objectTable.InsertRange(0, buffer); + return buffer.ToArray(); + } + + private static byte[] writeBinaryInteger(int value, bool write) + { + List buffer = new List(BitConverter.GetBytes((long)value)); + buffer = new List(RegulateNullBytes(buffer.ToArray())); + while (buffer.Count != Math.Pow(2, Math.Log(buffer.Count) / Math.Log(2))) + buffer.Add(0); + int header = 0x10 | (int)(Math.Log(buffer.Count) / Math.Log(2)); + + buffer.Reverse(); + + buffer.Insert(0, Convert.ToByte(header)); + + if (write) + objectTable.InsertRange(0, buffer); + + return buffer.ToArray(); + } + + private static byte[] writeBinaryDouble(double value) + { + List buffer = new List(RegulateNullBytes(BitConverter.GetBytes(value), 4)); + while (buffer.Count != Math.Pow(2, Math.Log(buffer.Count) / Math.Log(2))) + buffer.Add(0); + int header = 0x20 | (int)(Math.Log(buffer.Count) / Math.Log(2)); + + buffer.Reverse(); + + buffer.Insert(0, Convert.ToByte(header)); + + objectTable.InsertRange(0, buffer); + + return buffer.ToArray(); + } + + private static byte[] writeBinaryByteArray(byte[] value) + { + List buffer = new List(value); + List header = new List(); + if (value.Length < 15) + { + header.Add(Convert.ToByte(0x40 | Convert.ToByte(value.Length))); + } + else + { + header.Add(0x40 | 0xf); + header.AddRange(writeBinaryInteger(buffer.Count, false)); + } + + buffer.InsertRange(0, header); + + objectTable.InsertRange(0, buffer); + + return buffer.ToArray(); + } + + private static byte[] writeBinaryString(string value, bool head) + { + List buffer = new List(); + List header = new List(); + foreach (char chr in value.ToCharArray()) + buffer.Add(Convert.ToByte(chr)); + + if (head) + { + if (value.Length < 15) + { + header.Add(Convert.ToByte(0x50 | Convert.ToByte(value.Length))); + } + else + { + header.Add(0x50 | 0xf); + header.AddRange(writeBinaryInteger(buffer.Count, false)); + } + } + + buffer.InsertRange(0, header); + + objectTable.InsertRange(0, buffer); + + return buffer.ToArray(); + } + + private static byte[] RegulateNullBytes(byte[] value) + { + return RegulateNullBytes(value, 1); + } + + private static byte[] RegulateNullBytes(byte[] value, int minBytes) + { + Array.Reverse(value); + List bytes = new List(value); + for (int i = 0; i < bytes.Count; i++) + { + if (bytes[i] == 0 && bytes.Count > minBytes) + { + bytes.Remove(bytes[i]); + i--; + } + else + break; + } + + if (bytes.Count < minBytes) + { + int dist = minBytes - bytes.Count; + for (int i = 0; i < dist; i++) + bytes.Insert(0, 0); + } + + value = bytes.ToArray(); + Array.Reverse(value); + return value; + } + + private static void parseTrailer(List trailer) + { + offsetByteSize = BitConverter.ToInt32(RegulateNullBytes(trailer.GetRange(6, 1).ToArray(), 4), 0); + objRefSize = BitConverter.ToInt32(RegulateNullBytes(trailer.GetRange(7, 1).ToArray(), 4), 0); + byte[] refCountBytes = trailer.GetRange(12, 4).ToArray(); + Array.Reverse(refCountBytes); + refCount = BitConverter.ToInt32(refCountBytes, 0); + byte[] offsetTableOffsetBytes = trailer.GetRange(24, 8).ToArray(); + Array.Reverse(offsetTableOffsetBytes); + offsetTableOffset = BitConverter.ToInt64(offsetTableOffsetBytes, 0); + } + + private static void parseOffsetTable(List offsetTableBytes) + { + for (int i = 0; i < offsetTableBytes.Count; i += offsetByteSize) + { + byte[] buffer = offsetTableBytes.GetRange(i, offsetByteSize).ToArray(); + Array.Reverse(buffer); + offsetTable.Add(BitConverter.ToInt32(RegulateNullBytes(buffer, 4), 0)); + } + } + + private static object parseBinaryDictionary(int objRef) + { + Dictionary buffer = new Dictionary(); + List refs = new List(); + int refCount = 0; + + int refStartPosition; + refCount = getCount(offsetTable[objRef], out refStartPosition); + + + if (refCount < 15) + refStartPosition = offsetTable[objRef] + 1; + else + refStartPosition = offsetTable[objRef] + 2 + RegulateNullBytes(BitConverter.GetBytes(refCount), 1).Length; + + for (int i = refStartPosition; i < refStartPosition + refCount * 2 * objRefSize; i += objRefSize) + { + byte[] refBuffer = objectTable.GetRange(i, objRefSize).ToArray(); + Array.Reverse(refBuffer); + refs.Add(BitConverter.ToInt32(RegulateNullBytes(refBuffer, 4), 0)); + } + + for (int i = 0; i < refCount; i++) + { + buffer.Add((string)parseBinary(refs[i]), parseBinary(refs[i + refCount])); + } + + return buffer; + } + + private static object parseBinaryArray(int objRef) + { + List buffer = new List(); + List refs = new List(); + int refCount = 0; + + int refStartPosition; + refCount = getCount(offsetTable[objRef], out refStartPosition); + + + if (refCount < 15) + refStartPosition = offsetTable[objRef] + 1; + else + //The following integer has a header aswell so we increase the refStartPosition by two to account for that. + refStartPosition = offsetTable[objRef] + 2 + RegulateNullBytes(BitConverter.GetBytes(refCount), 1).Length; + + for (int i = refStartPosition; i < refStartPosition + refCount * objRefSize; i += objRefSize) + { + byte[] refBuffer = objectTable.GetRange(i, objRefSize).ToArray(); + Array.Reverse(refBuffer); + refs.Add(BitConverter.ToInt32(RegulateNullBytes(refBuffer, 4), 0)); + } + + for (int i = 0; i < refCount; i++) + { + buffer.Add(parseBinary(refs[i])); + } + + return buffer; + } + + private static int getCount(int bytePosition, out int newBytePosition) + { + byte headerByte = objectTable[bytePosition]; + byte headerByteTrail = Convert.ToByte(headerByte & 0xf); + int count; + if (headerByteTrail < 15) + { + count = headerByteTrail; + newBytePosition = bytePosition + 1; + } + else + count = (int)parseBinaryInt(bytePosition + 1, out newBytePosition); + return count; + } + + private static object parseBinary(int objRef) + { + byte header = objectTable[offsetTable[objRef]]; + switch (header & 0xF0) + { + case 0: + { + //If the byte is + //0 return null + //9 return true + //8 return false + return (objectTable[offsetTable[objRef]] == 0) ? (object)null : ((objectTable[offsetTable[objRef]] == 9) ? true : false); + } + case 0x10: + { + return parseBinaryInt(offsetTable[objRef]); + } + case 0x20: + { + return parseBinaryReal(offsetTable[objRef]); + } + case 0x30: + { + return parseBinaryDate(offsetTable[objRef]); + } + case 0x40: + { + return parseBinaryByteArray(offsetTable[objRef]); + } + case 0x50://String ASCII + { + return parseBinaryAsciiString(offsetTable[objRef]); + } + case 0x60://String Unicode + { + return parseBinaryUnicodeString(offsetTable[objRef]); + } + case 0xD0: + { + return parseBinaryDictionary(objRef); + } + case 0xA0: + { + return parseBinaryArray(objRef); + } + } + throw new Exception("This type is not supported"); + } + + public static object parseBinaryDate(int headerPosition) + { + byte[] buffer = objectTable.GetRange(headerPosition + 1, 8).ToArray(); + Array.Reverse(buffer); + double appleTime = BitConverter.ToDouble(buffer, 0); + DateTime result = PlistDateConverter.ConvertFromAppleTimeStamp(appleTime); + return result; + } + + private static object parseBinaryInt(int headerPosition) + { + int output; + return parseBinaryInt(headerPosition, out output); + } + + private static object parseBinaryInt(int headerPosition, out int newHeaderPosition) + { + byte header = objectTable[headerPosition]; + int byteCount = (int)Math.Pow(2, header & 0xf); + byte[] buffer = objectTable.GetRange(headerPosition + 1, byteCount).ToArray(); + Array.Reverse(buffer); + //Add one to account for the header byte + newHeaderPosition = headerPosition + byteCount + 1; + return BitConverter.ToInt32(RegulateNullBytes(buffer, 4), 0); + } + + private static object parseBinaryReal(int headerPosition) + { + byte header = objectTable[headerPosition]; + int byteCount = (int)Math.Pow(2, header & 0xf); + byte[] buffer = objectTable.GetRange(headerPosition + 1, byteCount).ToArray(); + Array.Reverse(buffer); + + return BitConverter.ToDouble(RegulateNullBytes(buffer, 8), 0); + } + + private static object parseBinaryAsciiString(int headerPosition) + { + int charStartPosition; + int charCount = getCount(headerPosition, out charStartPosition); + + var buffer = objectTable.GetRange(charStartPosition, charCount); + return buffer.Count > 0 ? Encoding.ASCII.GetString(buffer.ToArray()) : string.Empty; + } + + private static object parseBinaryUnicodeString(int headerPosition) + { + int charStartPosition; + int charCount = getCount(headerPosition, out charStartPosition); + charCount = charCount * 2; + + byte[] buffer = new byte[charCount]; + byte one, two; + + for (int i = 0; i < charCount; i += 2) + { + one = objectTable.GetRange(charStartPosition + i, 1)[0]; + two = objectTable.GetRange(charStartPosition + i + 1, 1)[0]; + + if (BitConverter.IsLittleEndian) + { + buffer[i] = two; + buffer[i + 1] = one; + } + else + { + buffer[i] = one; + buffer[i + 1] = two; + } + } + + return Encoding.Unicode.GetString(buffer); + } + + private static object parseBinaryByteArray(int headerPosition) + { + int byteStartPosition; + int byteCount = getCount(headerPosition, out byteStartPosition); + return objectTable.GetRange(byteStartPosition, byteCount).ToArray(); + } + + #endregion + } + + public enum plistType + { + Auto, Binary, Xml + } + + public static class PlistDateConverter + { + public static long timeDifference = 978307200; + + public static long GetAppleTime(long unixTime) + { + return unixTime - timeDifference; + } + + public static long GetUnixTime(long appleTime) + { + return appleTime + timeDifference; + } + + public static DateTime ConvertFromAppleTimeStamp(double timestamp) + { + DateTime origin = new DateTime(2001, 1, 1, 0, 0, 0, 0); + return origin.AddSeconds(timestamp); + } + + public static double ConvertToAppleTimeStamp(DateTime date) + { + DateTime begin = new DateTime(2001, 1, 1, 0, 0, 0, 0); + TimeSpan diff = date - begin; + return Math.Floor(diff.TotalSeconds); + } + } +} diff --git a/RepoFullDownloader-Core/Program.cs b/RepoFullDownloader-Core/Program.cs index ad6c034..e7475e9 100644 --- a/RepoFullDownloader-Core/Program.cs +++ b/RepoFullDownloader-Core/Program.cs @@ -1,12 +1,15 @@ using ICSharpCode.SharpZipLib.BZip2; using ICSharpCode.SharpZipLib.GZip; +using PlistCS; + using RepoFullDownloader; using System; using System.Collections.Generic; using System.IO; using System.Net; +using System.Net.Http; using System.Text.Json; using System.Threading; @@ -20,12 +23,25 @@ namespace RepoFullDownloader_Core { Console.WriteLine("RepoFullDownloader by KawaiiZenbo"); + ServicePointManager.ServerCertificateValidationCallback += (sender, certificate, chain, errors) => + { + // security + return true; + }; // Initial Checks if (!Directory.Exists("./output/")) { Directory.CreateDirectory("./output/"); } // Load Options from 'options.json' + if (!File.Exists("./options.json")) + { + Console.WriteLine("Could not find options.json"); + Console.WriteLine("Generating example..."); + // generate example options + File.WriteAllText("./options.json", JsonSerializer.Serialize(new Options())); + return; + } string optionsJson = File.ReadAllText("./options.json"); options = JsonSerializer.Deserialize(optionsJson); if (args.Length != 0) @@ -41,7 +57,7 @@ namespace RepoFullDownloader_Core } try { - DownloadRepo(url, options.originalFilenames); + DownloadRepo(url); } catch(Exception) { @@ -51,24 +67,29 @@ namespace RepoFullDownloader_Core } else { - if (!File.Exists("./options.json")) - { - Console.WriteLine("Could not find options.json"); - Console.WriteLine("Generating example..."); - // generate example options - File.WriteAllText("./options.json", JsonSerializer.Serialize(new Options())); - return; - } foreach (Repo r in options.repos) { - if (r.isInstaller) + if (r.type.ToLower() == "installer") { DownloadInstallerRepo(r.url); } + else if (r.type.ToLower() == "cydia") + { + DownloadRepo(r.url); + } + else if (r.type.ToLower() == "dist") + { + if(r.distAttributes == null) + { + Console.WriteLine($"No dist attributes were defined for {r.url} :("); + return; + } + DownloadDistRepo(r.url, r.distAttributes); + } else { - DownloadRepo(r.url, options.originalFilenames); + Console.WriteLine($"Invalid repo type {r.type} on {r.url}"); } } } @@ -77,7 +98,7 @@ namespace RepoFullDownloader_Core Console.WriteLine("done :)"); } - static void DownloadRepo(string link, bool keepOg) + static void DownloadRepo(string link) { if (!link.StartsWith("https://") && !link.StartsWith("http://")) { @@ -89,18 +110,17 @@ namespace RepoFullDownloader_Core } string cleanLink = link.TrimEnd('/').Replace("http://", "").Replace("https://", "").Replace("/", "_").Replace(":", "_"); Directory.CreateDirectory($"./output/{cleanLink}"); - WebClient webClient = new WebClient(); + HttpClient webClient = new HttpClient(); // headers because some repos are 'interesting' - webClient.Headers.Add("X-Machine", "iPod4,1"); - webClient.Headers.Add("X-Unique-ID", "0000000000000000000000000000000000000000"); - webClient.Headers.Add("X-Firmware", "6.1"); - webClient.Headers.Add("User-Agent", "Telesphoreo APT-HTTP/1.0.999"); + webClient.DefaultRequestHeaders.Add("X-Machine", "iPod4,1"); + webClient.DefaultRequestHeaders.Add("X-Unique-ID", "0000000000000000000000000000000000000000"); + webClient.DefaultRequestHeaders.Add("X-Firmware", "6.1"); + webClient.DefaultRequestHeaders.Add("User-Agent", "Telesphoreo APT-HTTP/1.0.999"); // Attempt to download packages file (try/catch hell) try { Console.WriteLine("Attempting to download " + link + "Packages.bz2"); - webClient.DownloadFile(new Uri(link + "Packages.bz2"), $"./output/{cleanLink}/Packages.bz2"); - FileStream packagesBz2 = new FileInfo($"./output/{cleanLink}/Packages.bz2").OpenRead(); + Stream packagesBz2 = webClient.GetStreamAsync(link + "Packages.bz2").Result; FileStream packagesBz2Decompressed = File.Create($"./output/{cleanLink}/Packages"); BZip2.Decompress(packagesBz2, packagesBz2Decompressed, true); } @@ -110,8 +130,7 @@ namespace RepoFullDownloader_Core { Console.WriteLine("Could not download " + link + "Packages.bz2: " + e.Message); Console.WriteLine("Attempting to download " + link + "Packages.gz"); - webClient.DownloadFile(new Uri(link + "Packages.gz"), $"./output/{cleanLink}/Packages.gz"); - FileStream packagesGz = new FileInfo($"./output/{cleanLink}/Packages.gz").OpenRead(); + Stream packagesGz = webClient.GetStreamAsync(link + "Packages.bz2").Result; FileStream packagesGzDecompressed = File.Create($"./output/{cleanLink}/Packages"); GZip.Decompress(packagesGz, packagesGzDecompressed, true); } @@ -121,7 +140,10 @@ namespace RepoFullDownloader_Core { Console.WriteLine("Could not download " + link + "Packages.gz: " + _e.Message); Console.WriteLine("Attempting to download " + link + "Packages"); - webClient.DownloadFile(new Uri(link + "Packages"), $"./output/{cleanLink}/Packages"); + using (StreamWriter outputFile = new StreamWriter($"./output/{cleanLink}/Packages")) + { + outputFile.WriteLine(webClient.GetStreamAsync(link + "Packages").Result); + } } catch (Exception __e) { @@ -167,12 +189,15 @@ namespace RepoFullDownloader_Core try { string[] choppedUp = p.link.Split('/'); - string fileToDownload = keepOg ? $"./output/{cleanLink}/" + choppedUp[choppedUp.Length - 1] : $"./output/{cleanLink}/" + p.name + "-" + p.version + ".deb"; + string fileToDownload = options.originalFilenames ? $"./output/{cleanLink}/" + choppedUp[choppedUp.Length - 1].Replace("/", "_").Replace(":", "_") : $"./output/{cleanLink}/" + p.name.Replace("/", "_").Replace(":", "_") + "-" + p.version.Replace("/", "_").Replace(":", "_") + ".deb"; if (File.Exists(fileToDownload)) { fileToDownload += "_" + r.Next(0000, 9999); } - webClient.DownloadFile(new Uri(link + p.link), fileToDownload); + using (WebClient wc = new WebClient()) + { + wc.DownloadFileAsync(new Uri(link + p.link), fileToDownload); + } Console.WriteLine("Successfully downloaded " + link + p.link + " as " + fileToDownload); } catch (Exception e) @@ -197,49 +222,192 @@ namespace RepoFullDownloader_Core { link += "/"; } - string cleanLink = link.Replace("http://", "").Replace("/", "_"); + string cleanLink = link.TrimEnd('/').Replace("http://", "").Replace("https://", "").Replace("/", "_").Replace(":", "_"); + Directory.CreateDirectory($"./output/{cleanLink}"); WebClient webClient = new WebClient(); + webClient.Headers.Add("User-Agent", "AppTapp Installer/3.0 (iPhone/1.1, like CFNetwork/100.0)"); try { Console.WriteLine("Attempting to download installer repo " + link); - webClient.DownloadFile(new Uri(link), $"./output/{cleanLink}/packages.xml"); + webClient.DownloadFile(link, $"./output/{cleanLink}/packages.plist"); } catch (Exception e) { Console.WriteLine("Could not download package list from " + link + ": " + e.Message); return; } - List plist = new List(File.ReadAllLines($"./output/{cleanLink}/packages.xml")); - List packages = new List(); - int i = 1; - foreach (string s in plist) - { - if (s.Contains("ocation")) - { - packages.Add(plist[i].Split('<')[1].Remove(0, 7)); - } - i++; - } - foreach (string s in packages) + Dictionary plist = (Dictionary)Plist.readPlist($"./output/{cleanLink}/packages.plist"); + foreach (Dictionary d in (List)plist["packages"]) { Random r = new Random(); try { - string[] choppedUp = s.Split('/'); + string[] choppedUp = d["location"].ToString().Split('/'); string fileToDownload = $"./output/{cleanLink}/" + choppedUp[choppedUp.Length - 1]; if (File.Exists(fileToDownload)) { fileToDownload += "_" + r.Next(0000, 9999); } - webClient.DownloadFile(new Uri(s), fileToDownload); - Console.WriteLine("Successfully downloaded " + s); + webClient.DownloadFile(new Uri((string)d["location"]), fileToDownload); + Console.WriteLine("Successfully downloaded " + (string)d["location"]); + } + catch (KeyNotFoundException) + { + string[] choppedUp = d["url"].ToString().Split('/'); + string fileToDownload = $"./output/{cleanLink}/" + choppedUp[choppedUp.Length - 1]; + if (File.Exists(fileToDownload)) + { + fileToDownload += "_" + r.Next(0000, 9999); + } + webClient.DownloadFile(new Uri((string)d["url"]), fileToDownload); + Console.WriteLine("Successfully downloaded " + (string)d["url"]); + Dictionary pl = (Dictionary)Plist.readPlist(fileToDownload); + { + string[] choppedUp2 = pl["location"].ToString().Split('/'); + string fileToDownload2 = $"./output/{cleanLink}/" + choppedUp2[choppedUp2.Length - 1]; + if (File.Exists(fileToDownload2)) + { + fileToDownload2 += "_" + r.Next(0000, 9999); + } + webClient.DownloadFile(new Uri((string)pl["location"]), fileToDownload2); + Console.WriteLine("Successfully downloaded " + (string)pl["location"]); + } } catch (Exception e) { - Console.WriteLine("Could not download " + s); + Console.WriteLine("Could not download " + (string)d["location"]); Console.WriteLine(e.Message); } } } + + static void DownloadDistRepo(string link, DistAttributes da) + { + // clean up link so no issues can ever arise + if (!link.StartsWith("https://") && !link.StartsWith("http://")) + { + link = "http://" + link; + } + if (!link.EndsWith("/")) + { + link += "/"; + } + + // make that good dist path + string distPath; + if(!da.suites.EndsWith("/")) + { + da.suites += "/"; + } + if (!da.components.EndsWith("/")) + { + da.components += "/"; + } + distPath = da.suites + da.components; + string poolpfLink = link + "dists/" + distPath + "binary-iphoneos-arm/"; + + string cleanLink = link.TrimEnd('/').Replace("http://", "").Replace("https://", "").Replace("/", "_").Replace(":", "_"); + Directory.CreateDirectory($"./output/{cleanLink}-({distPath.Replace("/", "_").Replace(":", "_")})"); + HttpClient webClient = new HttpClient(); + // headers because some repos are 'interesting' + webClient.DefaultRequestHeaders.Add("X-Machine", "iPod4,1"); + webClient.DefaultRequestHeaders.Add("X-Unique-ID", "0000000000000000000000000000000000000000"); + webClient.DefaultRequestHeaders.Add("X-Firmware", "6.1"); + webClient.DefaultRequestHeaders.Add("User-Agent", "Telesphoreo APT-HTTP/1.0.999"); + // Attempt to download packages file (try/catch hell) + try + { + Console.WriteLine("Attempting to download " + poolpfLink + "Packages.bz2"); + Stream packagesBz2 = webClient.GetStreamAsync(poolpfLink + "Packages.bz2").Result; + FileStream packagesBz2Decompressed = File.Create($"./output/{cleanLink}-({distPath.Replace("/", "_").Replace(":", "_")})/Packages"); + BZip2.Decompress(packagesBz2, packagesBz2Decompressed, true); + } + catch (Exception e) + { + try + { + Console.WriteLine("Could not download " + poolpfLink + "Packages.bz2: " + e.Message); + Console.WriteLine("Attempting to download " + poolpfLink + "Packages.gz"); + Stream packagesGz = webClient.GetStreamAsync(poolpfLink + "Packages.gz").Result; + FileStream packagesGzDecompressed = File.Create($"./output/{cleanLink}-({distPath.Replace("/", "_").Replace(":", "_")})/Packages"); + GZip.Decompress(packagesGz, packagesGzDecompressed, true); + } + catch (Exception _e) + { + try + { + Console.WriteLine("Could not download " + poolpfLink + "Packages.gz: " + _e.Message); + Console.WriteLine("Attempting to download " + poolpfLink + "Packages"); + using (StreamWriter outputFile = new StreamWriter($"./output/{cleanLink}-({distPath.Replace("/", "_").Replace(":", "_")})/Packages")) + { + outputFile.WriteLine(webClient.GetStreamAsync(poolpfLink + "Packages").Result); + } + } + catch (Exception __e) + { + Console.WriteLine("Could not download " + poolpfLink + "Packages: " + __e.Message); + Console.WriteLine("Could not locate packages file in " + poolpfLink); + Console.WriteLine(__e.Message); + throw; + } + } + } + Thread.Sleep(500); + // Clean list of package links, names, and versions + List packages = new List(); + foreach (string s in File.ReadAllText($"./output/{cleanLink}-({distPath.Replace("/", "_").Replace(":", "_")})/Packages").Split("\n\n")) + { + string name = ""; + string version = ""; + string _link = ""; + foreach (string s2 in s.Split('\n')) + { + if (s2.StartsWith("Package: ")) + { + name = s2.Remove(0, 8).Trim(); + } + else if (s2.StartsWith("Version: ")) + { + version = s2.Remove(0, 8).Trim(); + } + else if (s2.StartsWith("Filename: ")) + { + _link = s2.Remove(0, 9).Trim(); + } + } + packages.Add(new CydiaPackage(_link, name, version)); + } + // remove last one because ???? + packages.RemoveAt(packages.Count - 1); + List failed = new List(); + foreach (CydiaPackage p in packages) + { + // Download all packages on repo + Random r = new Random(); + try + { + string[] choppedUp = p.link.Split('/'); + string fileToDownload = options.originalFilenames ? $"./output/{cleanLink}-({distPath.Replace("/", "_").Replace(":", "_")})/" + choppedUp[choppedUp.Length - 1].Replace("/", "_").Replace(":", "_") : $"./output/{cleanLink}-({distPath.Replace("/", "_").Replace(":", "_")})/" + p.name.Replace("/", "_").Replace(":", "_") + "-" + p.version.Replace("/", "_").Replace(":", "_") + ".deb"; + if (File.Exists(fileToDownload)) + { + fileToDownload += "_" + r.Next(0000, 9999); + } + using (WebClient wc = new WebClient()) + { + wc.DownloadFileAsync(new Uri(link + p.link), fileToDownload); + } + Console.WriteLine("Successfully downloaded " + link + p.link + " as " + fileToDownload); + } + catch (Exception e) + { + Console.WriteLine("Could not download " + link + p.link); + Console.WriteLine(e.Message); + failed.Add(link + p.link); + } + Thread.Sleep(options.delay); + } + Console.WriteLine("Finished downloading " + link); + if (failed.Count != 0) File.WriteAllLines($"./output/{cleanLink}-({distPath.Replace("/", "_").Replace(":", "_")})/failed.txt", failed); + } } } diff --git a/example-options.json b/example-options.json index 11969b0..981b945 100644 --- a/example-options.json +++ b/example-options.json @@ -1,13 +1,25 @@ { - "originalFilenames": false, - "repos": [ + "originalFilenames":false, + "repos":[ { - "url": "http://cydia.invoxiplaygames.uk/", - "isInstaller": false + "url":"http://repo.kawaiizenbo.me/", + "type":"cydia", + "distAttributes":null }, { - "url": "http://apptapp.saurik.com", - "isInstaller": true + "url":"http://apptapp.saurik.com/", + "type":"installer", + "distAttributes":null + }, + { + "url":"http://apt.saurik.com/", + "type":"dist", + "distAttributes": + { + "suites":"ios/", + "components":"main" + } } - ] -} + ], + "delay":1 +} \ No newline at end of file