Crunchy-DL
/
Crunchy-Downloader
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add - Added download speed limit to settings 

Chg - Replace special characters in name with nothing instead of _
Chg - Adjusted estimated time calculation
Chg - Adjusted download speed calculation
Fix - Initial calendar loading didn't work always
This commit is contained in:
Elwador 2024-07-06 20:48:58 +02:00
parent baf90f546b
commit aee8a20450
10 changed files with 184 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
CRD/Downloader/Crunchyroll.cs
View File

@ -217,7 +217,7 @@ public class Crunchyroll{
UpdateDownloadListItems();
}
public void UpdateDownloadListItems(){
var list = Queue;
@ -243,8 +243,13 @@ public class Crunchyroll{
return forDate;
}
var request = HttpClientReq.CreateRequestMessage($"{calendarLanguage[CrunOptions.SelectedCalendarLanguage ?? "de"]}?filter=premium&date={weeksMondayDate}", HttpMethod.Get, true, true, null);
var request = HttpClientReq.CreateRequestMessage($"{calendarLanguage[CrunOptions.SelectedCalendarLanguage ?? "de"]}?filter=premium&date={weeksMondayDate}", HttpMethod.Get, false, false, null);
request.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36");
request.Headers.AcceptEncoding.ParseAdd("gzip, deflate");
request.Headers.AcceptLanguage.ParseAdd("en-US,en;q=0.9");
request.Headers.Referrer = new Uri("https://www.crunchyroll.com/");
var response = await HttpClientReq.Instance.SendHttpRequest(request);
CalendarWeek week = new CalendarWeek();

2
CRD/Utils/Files/FileNameManager.cs
View File

@ -89,7 +89,7 @@ public class FileNameManager{
}
public static string CleanupFilename(string filename){
string fixingChar = "_";
string fixingChar = "";
Regex illegalRe = new Regex(@"[\/\?<>\\:\*\|"":]"); // Illegal Characters on most Operating Systems
Regex controlRe = new Regex(@"[\x00-\x1f\x80-\x9f]"); // Unicode Control codes: C0 and C1
Regex reservedRe = new Regex(@"^\.\.?$"); // Reserved filenames on Unix-based systems (".", "..")

50
CRD/Utils/HLS/HLSDownloader.cs
View File

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Security.Cryptography;
using System.Text.RegularExpressions;
@ -71,11 +72,9 @@ public class HlsDownloader{
_data.Offset = resumeData.Completed;
_data.IsResume = true;
} else{
if (resumeData.Total == _data.M3U8Json?.Segments.Count &&
resumeData.Completed == resumeData.Total &&
!double.IsNaN(resumeData.Completed)){
Console.WriteLine("Already finished");
return (Ok: true, _data.Parts);
}
@ -118,12 +117,11 @@ public class HlsDownloader{
}
// Start time
_data.DateStart = DateTimeOffset.Now.ToUnixTimeMilliseconds();
if (_data.M3U8Json != null){
List<dynamic> segments = _data.M3U8Json.Segments;
// map has init uri outside is none init uri
// Download init part
if (segments[0].map != null && _data.Offset == 0 && !_data.SkipInit){
@ -158,6 +156,8 @@ public class HlsDownloader{
}
for (int p = 0; p < Math.Ceiling((double)segments.Count / _data.Threads); p++){
// Start time
_data.DateStart = DateTimeOffset.Now.ToUnixTimeMilliseconds();
int offset = p * _data.Threads;
int dlOffset = Math.Min(offset + _data.Threads, segments.Count);
@ -248,14 +248,15 @@ public class HlsDownloader{
int downloadedSeg = Math.Min(dlOffset, totalSeg);
_data.Parts.Completed = downloadedSeg + _data.Offset; //
var dataLog = GetDownloadInfo(_data.DateStart, _data.Parts.Completed, totalSeg, _data.BytesDownloaded);
var dataLog = GetDownloadInfo(_data.DateStart, _data.Parts.Completed, totalSeg, _data.BytesDownloaded,_data.TotalBytes);
_data.BytesDownloaded = 0;
// Save resume data to file
string resumeDataJson = JsonConvert.SerializeObject(new{ _data.Parts.Completed, Total = totalSeg });
File.WriteAllText($"{fn}.resume", resumeDataJson);
// Log progress
Console.WriteLine($"{_data.Parts.Completed} of {totalSeg} parts downloaded [{dataLog.Percent}%] ({FormatTime(dataLog.Time / 1000)} | {dataLog.DownloadSpeed / 1000000.0:F2}Mb/s)");
Console.WriteLine($"{_data.Parts.Completed} of {totalSeg} parts downloaded [{dataLog.Percent}%] ({FormatTime(dataLog.Time)} | {dataLog.DownloadSpeed / 1000000.0:F2}Mb/s)");
_currentEpMeta.DownloadProgress = new DownloadProgress(){
IsDownloading = true,
@ -283,7 +284,7 @@ public class HlsDownloader{
return (Ok: true, _data.Parts);
}
public static Info GetDownloadInfo(long dateStartUnix, int partsDownloaded, int partsTotal, long downloadedBytes){
public static Info GetDownloadInfo(long dateStartUnix, int partsDownloaded, int partsTotal, long downloadedBytes,long totalDownloadedBytes){
// Convert Unix timestamp to DateTime
DateTime dateStart = DateTimeOffset.FromUnixTimeMilliseconds(dateStartUnix).UtcDateTime;
double dateElapsed = (DateTime.UtcNow - dateStart).TotalMilliseconds;
@ -291,13 +292,15 @@ public class HlsDownloader{
// Calculate percentage
int percentFixed = (int)((double)partsDownloaded / partsTotal * 100);
int percent = percentFixed < 100 ? percentFixed : (partsTotal == partsDownloaded ? 100 : 99);
// Calculate remaining time estimate
double remainingTime = dateElapsed * (partsTotal / (double)partsDownloaded - 1);
// Calculate download speed (bytes per second)
double downloadSpeed = downloadedBytes / (dateElapsed / 1000);
// Calculate remaining time estimate
// double remainingTime = dateElapsed * (partsTotal / (double)partsDownloaded - 1);
int partsLeft = partsTotal - partsDownloaded;
double remainingTime = (partsLeft * (totalDownloadedBytes / partsDownloaded)) / downloadSpeed;
return new Info{
Percent = percent,
Time = remainingTime,
@ -324,11 +327,17 @@ public class HlsDownloader{
if (partContent != null) dec = decipher.TransformFinalBlock(partContent, 0, partContent.Length);
}
if (dec != null) _data.BytesDownloaded += dec.Length;
if (dec != null){
_data.BytesDownloaded += dec.Length;
_data.TotalBytes += dec.Length;
}
} else{
part = await GetData(p, sUri, seg.ByteRange != null ? seg.ByteRange.ToDictionary() : new Dictionary<string, string>(), segOffset, false, _data.Timeout, _data.Retries);
dec = part;
if (dec != null) _data.BytesDownloaded += dec.Length;
if (dec != null){
_data.BytesDownloaded += dec.Length;
_data.TotalBytes += dec.Length;
}
}
} catch (Exception ex){
throw new Exception($"Error at segment {p}: {ex.Message}", ex);
@ -428,7 +437,7 @@ public class HlsDownloader{
for (int attempt = 0; attempt < retryCount + 1; attempt++){
using (var request = CloneHttpRequestMessage(requestPara)){
try{
response = await HttpClientReq.Instance.GetHttpClient().SendAsync(request, HttpCompletionOption.ResponseContentRead);
response = await HttpClientReq.Instance.GetHttpClient().SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
response.EnsureSuccessStatusCode();
return await ReadContentAsByteArrayAsync(response.Content);
} catch (HttpRequestException ex){
@ -448,8 +457,14 @@ public class HlsDownloader{
private async Task<byte[]> ReadContentAsByteArrayAsync(HttpContent content){
using (var memoryStream = new MemoryStream())
using (var contentStream = await content.ReadAsStreamAsync()){
await contentStream.CopyToAsync(memoryStream, 81920);
using (var contentStream = await content.ReadAsStreamAsync())
using (var throttledStream = new ThrottledStream(contentStream)){
byte[] buffer = new byte[8192];
int bytesRead;
while ((bytesRead = await throttledStream.ReadAsync(buffer, 0, buffer.Length)) > 0){
await memoryStream.WriteAsync(buffer, 0, bytesRead);
}
return memoryStream.ToArray();
}
}
@ -567,6 +582,7 @@ public class Data{
public int WaitTime{ get; set; }
public string? Override{ get; set; }
public long DateStart{ get; set; }
public long TotalBytes{ get; set; }
}
public class ProgressData{

97
CRD/Utils/HLS/ThrottledStream.cs Normal file
View File

@ -0,0 +1,97 @@
using CRD.Downloader;
namespace CRD.Utils.HLS;
using System;
using System.IO;
using System.Threading;
public class GlobalThrottler{
private static GlobalThrottler _instance;
private static readonly object _lock = new object();
private long _totalBytesRead;
private DateTime _lastReadTime;
private GlobalThrottler(){
_totalBytesRead = 0;
_lastReadTime = DateTime.Now;
}
public static GlobalThrottler Instance(){
if (_instance == null){
lock (_lock){
if (_instance == null){
_instance = new GlobalThrottler();
}
}
}
return _instance;
}
public void Throttle(int bytesRead){
if (Crunchyroll.Instance.CrunOptions.DownloadSpeedLimit == 0) return;
lock (_lock){
_totalBytesRead += bytesRead;
if (_totalBytesRead >= ((Crunchyroll.Instance.CrunOptions.DownloadSpeedLimit * 1024) / 10)){
var timeElapsed = DateTime.Now - _lastReadTime;
if (timeElapsed.TotalMilliseconds < 100){
Thread.Sleep(100 - (int)timeElapsed.TotalMilliseconds);
}
_totalBytesRead = 0;
_lastReadTime = DateTime.Now;
}
}
}
}
public class ThrottledStream : Stream{
private readonly Stream _baseStream;
private readonly GlobalThrottler _throttler;
public ThrottledStream(Stream baseStream){
_baseStream = baseStream ?? throw new ArgumentNullException(nameof(baseStream));
_throttler = GlobalThrottler.Instance();
}
public override bool CanRead => _baseStream.CanRead;
public override bool CanSeek => _baseStream.CanSeek;
public override bool CanWrite => _baseStream.CanWrite;
public override long Length => _baseStream.Length;
public override long Position{
get => _baseStream.Position;
set => _baseStream.Position = value;
}
public override void Flush() => _baseStream.Flush();
public override long Seek(long offset, SeekOrigin origin) => _baseStream.Seek(offset, origin);
public override void SetLength(long value) => _baseStream.SetLength(value);
public override void Write(byte[] buffer, int offset, int count) => _baseStream.Write(buffer, offset, count);
public override int Read(byte[] buffer, int offset, int count){
int bytesRead = 0;
if (Crunchyroll.Instance.CrunOptions.DownloadSpeedLimit != 0){
int bytesToRead = Math.Min(count, (Crunchyroll.Instance.CrunOptions.DownloadSpeedLimit * 1024) / 10);
bytesRead = _baseStream.Read(buffer, offset, bytesToRead);
_throttler.Throttle(bytesRead);
} else{
bytesRead = _baseStream.Read(buffer, offset, count);
}
return bytesRead;
}
protected override void Dispose(bool disposing){
if (disposing){
_baseStream.Dispose();
}
base.Dispose(disposing);
}
}

6
CRD/Utils/Http/HttpClientReq.cs
View File

@ -1,4 +1,4 @@
using System;
using System;
using System.Net;
using System.Net.Http;
using System.Collections.Generic;
@ -40,6 +40,7 @@ public class HttpClientReq{
handler = new HttpClientHandler();
handler.CookieContainer = new CookieContainer();
handler.UseCookies = true;
handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
// Initialize the HttpClient with the handler
client = new HttpClient(handler);
@ -95,8 +96,7 @@ public class HttpClientReq{
}
if (disableDrmHeader){
request.Headers.Add("X-Cr-Disable-Drm", "true");
request.Headers.Add("x-cr-stream-limits", "false");
}

3
CRD/Utils/Structs/CrDownloadOptions.cs
View File

@ -153,4 +153,7 @@ public class CrDownloadOptions{
[YamlMember(Alias = "history_page_properties", ApplyNamingConventions = false)]
public HistoryPageProperties? HistoryPageProperties{ get; set; }
[YamlMember(Alias = "download_speed_limit", ApplyNamingConventions = false)]
public int DownloadSpeedLimit{ get; set; }
}

4
CRD/Utils/Updater/Updater.cs
View File

@ -7,6 +7,7 @@ using System.Net.Http;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Threading.Tasks;
using CRD.Utils.HLS;
using CRD.ViewModels;
using CRD.Views.Utils;
using FluentAvalonia.UI.Controls;
@ -80,9 +81,6 @@ public class Updater : INotifyPropertyChanged{
public async Task DownloadAndUpdateAsync(){
try{
using (var client = new HttpClient()){
// Download the zip file

4
CRD/ViewModels/DownloadsPageViewModel.cs
View File

@ -70,7 +70,7 @@ public partial class DownloadItemModel : INotifyPropertyChanged{
Done = epMeta.DownloadProgress.Done;
Percent = epMeta.DownloadProgress.Percent;
Time = "Estimated Time: " + TimeSpan.FromSeconds(epMeta.DownloadProgress.Time / 1000).ToString(@"hh\:mm\:ss");
Time = "Estimated Time: " + TimeSpan.FromSeconds(epMeta.DownloadProgress.Time).ToString(@"hh\:mm\:ss");
DownloadSpeed = $"{epMeta.DownloadProgress.DownloadSpeed / 1000000.0:F2}Mb/s";
Paused = epMeta.Paused || !isDownloading && !epMeta.Paused;
DoingWhat = epMeta.Paused ? "Paused" : Done ? "Done" : epMeta.DownloadProgress.Doing != string.Empty ? epMeta.DownloadProgress.Doing : "Waiting";
@ -127,7 +127,7 @@ public partial class DownloadItemModel : INotifyPropertyChanged{
isDownloading = epMeta.DownloadProgress.IsDownloading || Done;
Done = epMeta.DownloadProgress.Done;
Percent = epMeta.DownloadProgress.Percent;
Time = "Estimated Time: " + TimeSpan.FromSeconds(epMeta.DownloadProgress.Time / 1000).ToString(@"hh\:mm\:ss");
Time = "Estimated Time: " + TimeSpan.FromSeconds(epMeta.DownloadProgress.Time).ToString(@"hh\:mm\:ss");
DownloadSpeed = $"{epMeta.DownloadProgress.DownloadSpeed / 1000000.0:F2}Mb/s";
Paused = epMeta.Paused || !isDownloading && !epMeta.Paused;

44
CRD/ViewModels/SettingsPageViewModel.cs
View File

@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Collections.Specialized;
@ -51,7 +51,7 @@ public partial class SettingsPageViewModel : ViewModelBase{
[ObservableProperty]
private bool _muxToMp4;
[ObservableProperty]
private bool _includeEpisodeDescription;
@ -65,14 +65,17 @@ public partial class SettingsPageViewModel : ViewModelBase{
private bool _history;
[ObservableProperty]
private int _leadingNumbers;
private double? _leadingNumbers;
[ObservableProperty]
private int _simultaneousDownloads;
private double? _simultaneousDownloads;
[ObservableProperty]
private double? _downloadSpeed;
[ObservableProperty]
private string _fileName = "";
[ObservableProperty]
private string _fileTitle = "";
@ -93,10 +96,10 @@ public partial class SettingsPageViewModel : ViewModelBase{
[ObservableProperty]
private ComboBoxItem _selectedHSLang;
[ObservableProperty]
private ComboBoxItem _selectedHistoryLang;
[ObservableProperty]
private ComboBoxItem _selectedDescriptionLang;
@ -249,7 +252,7 @@ public partial class SettingsPageViewModel : ViewModelBase{
new ComboBoxItem(){ Content = "hi-IN" },
new ComboBoxItem(){ Content = "ar-SA" },
};
public ObservableCollection<ComboBoxItem> DescriptionLangList{ get; } = new(){
new ComboBoxItem(){ Content = "default" },
new ComboBoxItem(){ Content = "de-DE" },
@ -264,7 +267,7 @@ public partial class SettingsPageViewModel : ViewModelBase{
new ComboBoxItem(){ Content = "hi-IN" },
new ComboBoxItem(){ Content = "ar-SA" },
};
public ObservableCollection<ComboBoxItem> DubLangList{ get; } = new(){
};
@ -296,7 +299,7 @@ public partial class SettingsPageViewModel : ViewModelBase{
new ComboBoxItem(){ Content = "android/phone" },
new ComboBoxItem(){ Content = "tv/samsung" },
};
[ObservableProperty]
private string _downloadDirPath;
@ -323,16 +326,16 @@ public partial class SettingsPageViewModel : ViewModelBase{
CrDownloadOptions options = Crunchyroll.Instance.CrunOptions;
DownloadDirPath = string.IsNullOrEmpty(options.DownloadDirPath) ? CfgManager.PathVIDEOS_DIR : options.DownloadDirPath;
ComboBoxItem? descriptionLang = DescriptionLangList.FirstOrDefault(a => a.Content != null && (string)a.Content == options.DescriptionLang) ?? null;
SelectedDescriptionLang = descriptionLang ?? DescriptionLangList[0];
ComboBoxItem? historyLang = HistoryLangList.FirstOrDefault(a => a.Content != null && (string)a.Content == options.HistoryLang) ?? null;
SelectedHistoryLang = historyLang ?? HistoryLangList[0];
ComboBoxItem? hsLang = HardSubLangList.FirstOrDefault(a => a.Content != null && (string)a.Content == options.Hslang) ?? null;
SelectedHSLang = hsLang ?? HardSubLangList[0];
ComboBoxItem? defaultDubLang = DefaultDubLangList.FirstOrDefault(a => a.Content != null && (string)a.Content == (options.DefaultAudio ?? "")) ?? null;
SelectedDefaultDubLang = defaultDubLang ?? DefaultDubLangList[0];
@ -371,6 +374,7 @@ public partial class SettingsPageViewModel : ViewModelBase{
AddScaledBorderAndShadow = options.SubsAddScaledBorder is ScaledBorderAndShadowSelection.ScaledBorderAndShadowNo or ScaledBorderAndShadowSelection.ScaledBorderAndShadowYes;
SelectedScaledBorderAndShadow = GetScaledBorderAndShadowFromOptions(options);
DownloadSpeed = options.DownloadSpeedLimit;
IncludeEpisodeDescription = options.IncludeVideoDescription;
FileTitle = options.VideoTitle ?? "";
IncludeSignSubs = options.IncludeSignsSubs;
@ -438,9 +442,11 @@ public partial class SettingsPageViewModel : ViewModelBase{
Crunchyroll.Instance.CrunOptions.Chapters = DownloadChapters;
Crunchyroll.Instance.CrunOptions.Mp4 = MuxToMp4;
Crunchyroll.Instance.CrunOptions.SkipSubsMux = SkipSubMux;
Crunchyroll.Instance.CrunOptions.Numbers = LeadingNumbers;
Crunchyroll.Instance.CrunOptions.Numbers = Math.Clamp((int)(LeadingNumbers ?? 0),0,10);
Crunchyroll.Instance.CrunOptions.FileName = FileName;
Crunchyroll.Instance.CrunOptions.IncludeSignsSubs = IncludeSignSubs;
Crunchyroll.Instance.CrunOptions.IncludeSignsSubs = IncludeSignSubs;
Crunchyroll.Instance.CrunOptions.DownloadSpeedLimit = Math.Clamp((int)(DownloadSpeed ?? 0),0,1000000000);
Crunchyroll.Instance.CrunOptions.SimultaneousDownloads = Math.Clamp((int)(SimultaneousDownloads ?? 0),1,10);
Crunchyroll.Instance.CrunOptions.SubsAddScaledBorder = GetScaledBorderAndShadowSelection();
@ -454,11 +460,11 @@ public partial class SettingsPageViewModel : ViewModelBase{
string descLang = SelectedDescriptionLang.Content + "";
Crunchyroll.Instance.CrunOptions.DescriptionLang = descLang != "default" ? descLang : "";
string historyLang = SelectedHistoryLang.Content + "";
Crunchyroll.Instance.CrunOptions.HistoryLang = historyLang != "default" ? historyLang : "";
string hslang = SelectedHSLang.Content + "";
Crunchyroll.Instance.CrunOptions.Hslang = hslang != "none" ? Languages.FindLang(hslang).Locale : hslang;
@ -477,7 +483,7 @@ public partial class SettingsPageViewModel : ViewModelBase{
Crunchyroll.Instance.CrunOptions.DubLang = dubLangs;
Crunchyroll.Instance.CrunOptions.SimultaneousDownloads = SimultaneousDownloads;
Crunchyroll.Instance.CrunOptions.QualityAudio = SelectedAudioQuality?.Content + "";
Crunchyroll.Instance.CrunOptions.QualityVideo = SelectedVideoQuality?.Content + "";

13
CRD/Views/SettingsPageView.axaml
View File

@ -150,6 +150,17 @@
IconSource="Download"
Description="Adjust download settings"
IsExpanded="False">
<controls:SettingsExpanderItem Content="Max Download Speed"
Description="Download in Kb/s - 0 is full speed">
<controls:SettingsExpanderItem.Footer>
<controls:NumberBox Minimum="0" Maximum="1000000000"
Value="{Binding DownloadSpeed}"
SpinButtonPlacementMode="Hidden"
HorizontalAlignment="Stretch" />
</controls:SettingsExpanderItem.Footer>
</controls:SettingsExpanderItem>
<controls:SettingsExpanderItem Content="Stream Endpoint ">
<controls:SettingsExpanderItem.Footer>
@ -181,7 +192,7 @@
<controls:SettingsExpanderItem Content="Simultaneous Downloads">
<controls:SettingsExpanderItem.Footer>
<controls:NumberBox Minimum="0" Maximum="5"
<controls:NumberBox Minimum="0" Maximum="10"
Value="{Binding SimultaneousDownloads}"
SpinButtonPlacementMode="Inline"
HorizontalAlignment="Stretch" />