Compare commits

...

5 Commits

Author SHA1 Message Date
Kelvin 570f32e980 PlatformUrl support 2023-11-03 15:39:27 +01:00
Kelvin 16a0351125 Per-plugin ratelimit setting 2023-11-03 15:15:18 +01:00
Kelvin 2fa9005806 Keep plugin settings on update 2023-11-03 14:46:43 +01:00
Kelvin 25527997fa Fix channels updating while they shouldnt 2023-11-03 14:37:36 +01:00
Kelvin 4655d8369d Reduce subscription calls, Improve subs sorting, Improve view sorting 2023-11-03 13:34:23 +01:00
31 changed files with 140 additions and 36 deletions
@@ -92,6 +92,19 @@ open class JSClient : IPlatformClient {
val enableInSearch get() = descriptor.appSettings.tabEnabled.enableSearch ?: true
val enableInHome get() = descriptor.appSettings.tabEnabled.enableHome ?: true
fun getSubscriptionRateLimit(): Int? {
val pluginRateLimit = config.subscriptionRateLimit;
val settingsRateLimit = descriptor.appSettings.rateLimit.getSubRateLimit();
if(settingsRateLimit > 0) {
if(pluginRateLimit != null)
return settingsRateLimit.coerceAtMost(pluginRateLimit);
else
return settingsRateLimit;
}
else
return pluginRateLimit;
}
val onDisabled = Event1<JSClient>();
val onCaptchaException = Event2<JSClient, ScriptCaptchaRequiredException>();
@@ -41,6 +41,7 @@ class SourcePluginConfig(
val constants: HashMap<String, String> = hashMapOf(),
//TODO: These should be vals...but prob for serialization reasons cannot be changed.
var platformUrl: String? = null,
var subscriptionRateLimit: Int? = null,
var enableInSearch: Boolean = true,
var enableInHome: Boolean = true,
@@ -3,6 +3,7 @@ package com.futo.platformplayer.api.media.platforms.js
import com.futo.platformplayer.R
import com.futo.platformplayer.constructs.Event0
import com.futo.platformplayer.serializers.FlexibleBooleanSerializer
import com.futo.platformplayer.views.fields.DropdownFieldOptions
import com.futo.platformplayer.views.fields.FieldForm
import com.futo.platformplayer.views.fields.FormField
import kotlinx.serialization.Serializable
@@ -79,6 +80,29 @@ class SourcePluginDescriptor {
var enableSearch: Boolean? = null;
}
@FormField(R.string.ratelimit, "group", R.string.ratelimit_description, 3)
var rateLimit = RateLimit();
@Serializable
class RateLimit {
@FormField(R.string.subscriptions, FieldForm.DROPDOWN, R.string.ratelimit_sub_setting_description, 1)
@DropdownFieldOptions("Plugin defined", "25", "50", "75", "100", "125", "150", "200")
var rateLimitSubs: Int = 0;
fun getSubRateLimit(): Int {
return when(rateLimitSubs) {
0 -> -1
1 -> 25
2 -> 50
3 -> 75
4 -> 100
5 -> 125
6 -> 150
7 -> 200
else -> -1
}
}
}
fun loadDefaults(config: SourcePluginConfig) {
@@ -176,8 +176,8 @@ class SubscriptionsFeedFragment : MainFragment() {
private val _taskGetPager = TaskHandler<Boolean, IPager<IPlatformContent>>({StateApp.instance.scope}, { withRefresh ->
if(!_bypassRateLimit) {
val subRequestCounts = StateSubscriptions.instance.getSubscriptionRequestCount();
val reqCountStr = subRequestCounts.map { " ${it.key.config.name}: ${it.value}/${it.key.config.subscriptionRateLimit}" }.joinToString("\n");
val rateLimitPlugins = subRequestCounts.filter { clientCount -> clientCount.key.config.subscriptionRateLimit?.let { rateLimit -> clientCount.value > rateLimit } == true }
val reqCountStr = subRequestCounts.map { " ${it.key.config.name}: ${it.value}/${it.key.getSubscriptionRateLimit()}" }.joinToString("\n");
val rateLimitPlugins = subRequestCounts.filter { clientCount -> clientCount.key.getSubscriptionRateLimit()?.let { rateLimit -> clientCount.value > rateLimit } == true }
Logger.w(TAG, "Refreshing subscriptions with requests:\n" + reqCountStr);
if(rateLimitPlugins.any())
throw RateLimitException(rateLimitPlugins.map { it.key.id });
@@ -610,7 +610,7 @@ class VideoDetailView : ConstraintLayout {
}
val _trackingUpdateTimeLock = Object();
val _trackingUpdateInterval = 3000;
val _trackingUpdateInterval = 2500;
var _trackingLastUpdateTime = System.currentTimeMillis();
var _trackingLastPosition: Long = 0;
var _trackingLastVideo: IPlatformVideoDetails? = null;
@@ -6,6 +6,7 @@ import com.futo.platformplayer.api.media.models.channels.SerializedChannel
import com.futo.platformplayer.api.media.models.contents.IPlatformContent
import com.futo.platformplayer.api.media.models.contents.IPlatformContentDetails
import com.futo.platformplayer.getNowDiffDays
import com.futo.platformplayer.logging.Logger
import com.futo.platformplayer.serializers.OffsetDateTimeSerializer
import com.futo.platformplayer.states.StatePlatform
import com.futo.platformplayer.states.StateSubscriptions
@@ -53,10 +54,12 @@ class Subscription {
this.channel = channel;
}
fun shouldFetchVideos() = true;
fun shouldFetchStreams() = doFetchStreams && lastLiveStream.getNowDiffDays() < 7;
fun shouldFetchLiveStreams() = doFetchLive && lastLiveStream.getNowDiffDays() < 14;
fun shouldFetchPosts() = doFetchPosts && lastPost.getNowDiffDays() < 2;
fun shouldFetchVideos() = doFetchVideos &&
(lastVideo.getNowDiffDays() < 30 || lastVideoUpdate.getNowDiffDays() >= 1) &&
(lastVideo.getNowDiffDays() < 180 || lastVideoUpdate.getNowDiffDays() >= 3);
fun shouldFetchStreams() = doFetchStreams && (lastLiveStream.getNowDiffDays() < 7);
fun shouldFetchLiveStreams() = doFetchLive && (lastLiveStream.getNowDiffDays() < 14);
fun shouldFetchPosts() = doFetchPosts && (lastPost.getNowDiffDays() < 5);
fun getClient() = StatePlatform.instance.getChannelClientOrNull(channel.url);
@@ -103,30 +106,39 @@ class Subscription {
else {
interval = 5;
mostRecent = null;
Logger.i("Subscription", "Subscription [${channel.name}]:${type} no results found");
}
when(type) {
ResultCapabilities.TYPE_VIDEOS -> {
uploadInterval = interval;
if(mostRecent != null)
lastVideo = mostRecent;
else if(lastVideo.year > 3000)
lastVideo = OffsetDateTime.MIN;
lastVideoUpdate = OffsetDateTime.now();
}
ResultCapabilities.TYPE_MIXED -> {
uploadInterval = interval;
if(mostRecent != null)
lastVideo = mostRecent;
else if(lastVideo.year > 3000)
lastVideo = OffsetDateTime.MIN;
lastVideoUpdate = OffsetDateTime.now();
}
ResultCapabilities.TYPE_STREAMS -> {
uploadStreamInterval = interval;
if(mostRecent != null)
lastLiveStream = mostRecent;
else if(lastLiveStream.year > 3000)
lastLiveStream = OffsetDateTime.MIN;
lastStreamUpdate = OffsetDateTime.now();
}
ResultCapabilities.TYPE_POSTS -> {
uploadPostInterval = interval;
if(mostRecent != null)
lastPost = mostRecent;
else if(lastPost.year > 3000)
lastPost = OffsetDateTime.MIN;
lastPostUpdate = OffsetDateTime.now();
}
}
@@ -452,8 +452,8 @@ class StateApp {
if(Settings.instance.subscriptions.fetchOnAppBoot) {
scope.launch(Dispatchers.IO) {
val subRequestCounts = StateSubscriptions.instance.getSubscriptionRequestCount();
val reqCountStr = subRequestCounts.map { " ${it.key.config.name}: ${it.value}/${it.key.config.subscriptionRateLimit}" }.joinToString("\n");
val isRateLimitReached = !subRequestCounts.any { clientCount -> clientCount.key.config.subscriptionRateLimit?.let { rateLimit -> clientCount.value > rateLimit } == true };
val reqCountStr = subRequestCounts.map { " ${it.key.config.name}: ${it.value}/${it.key.getSubscriptionRateLimit()}" }.joinToString("\n");
val isRateLimitReached = !subRequestCounts.any { clientCount -> clientCount.key.getSubscriptionRateLimit()?.let { rateLimit -> clientCount.value > rateLimit } == true };
if (isRateLimitReached) {
Logger.w(TAG, "Subscriptions request on boot, request counts:\n${reqCountStr}");
delay(5000);
@@ -374,7 +374,10 @@ class StatePlugins {
if(icon != null)
iconsDir.saveIconBinary(config.id, icon);
_plugins.save(SourcePluginDescriptor(config, existingAuth?.toEncrypted(), existingCaptcha?.toEncrypted(), flags));
val descriptor = SourcePluginDescriptor(config, existingAuth?.toEncrypted(), existingCaptcha?.toEncrypted(), flags);
descriptor.settings = existing?.settings ?: descriptor.settings;
descriptor.appSettings = existing?.appSettings ?: descriptor.appSettings;
_plugins.save(descriptor);
return null;
}
catch(ex: Throwable) {
@@ -59,7 +59,7 @@ class SmartSubscriptionAlgorithm(
for(clientTasks in ordering) {
val limit = clientTasks.first.config.subscriptionRateLimit;
val limit = clientTasks.first.getSubscriptionRateLimit();
if(limit == null || limit <= 0)
finalTasks.addAll(clientTasks.second);
else {
@@ -85,21 +85,21 @@ class SmartSubscriptionAlgorithm(
ResultCapabilities.TYPE_STREAMS -> sub.lastLiveStream;
ResultCapabilities.TYPE_LIVE -> sub.lastLiveStream;
ResultCapabilities.TYPE_POSTS -> sub.lastPost;
else -> sub.lastVideo; //TODO: minimum of all
else -> sub.lastVideo; //TODO: minimum of all?
};
val lastUpdate = when(type) {
ResultCapabilities.TYPE_VIDEOS -> sub.lastVideoUpdate;
ResultCapabilities.TYPE_STREAMS -> sub.lastLiveStreamUpdate;
ResultCapabilities.TYPE_LIVE -> sub.lastLiveStreamUpdate;
ResultCapabilities.TYPE_POSTS -> sub.lastPostUpdate;
else -> sub.lastVideoUpdate; //TODO: minimum of all
else -> sub.lastVideoUpdate; //TODO: minimum of all?
};
val interval = when(type) {
ResultCapabilities.TYPE_VIDEOS -> sub.uploadInterval;
ResultCapabilities.TYPE_STREAMS -> sub.uploadStreamInterval;
ResultCapabilities.TYPE_LIVE -> sub.uploadStreamInterval;
ResultCapabilities.TYPE_POSTS -> sub.uploadPostInterval;
else -> sub.uploadInterval; //TODO: minimum of all
else -> sub.uploadInterval; //TODO: minimum of all?
};
val lastItemDaysAgo = lastItem.getNowDiffHours();
val lastUpdateHoursAgo = lastUpdate.getNowDiffHours();
@@ -54,7 +54,7 @@ abstract class SubscriptionsTaskFetchAlgorithm(
val clientTaskCount = clientTasks.value.filter { !it.fromCache }.size;
val clientCacheCount = clientTasks.value.size - clientTaskCount;
if(clientCacheCount > 0) {
UIDialogs.toast("[${clientTasks.key.name}] only updating ${clientTaskCount} most urgent channels. (${clientCacheCount} cached)");
UIDialogs.toast("[${clientTasks.key.name}] only updating ${clientTaskCount} most urgent channels (rqs). (${clientCacheCount} cached)");
}
}
@@ -54,10 +54,10 @@ class SubscriptionAdapter : RecyclerView.Adapter<SubscriptionViewHolder> {
private fun updateDataset() {
_sortedDataset = when (sortBy) {
0 -> StateSubscriptions.instance.getSubscriptions().sortedBy({ u -> u.channel.name })
1 -> StateSubscriptions.instance.getSubscriptions().sortedByDescending({ u -> u.channel.name })
2 -> StateSubscriptions.instance.getSubscriptions().sortedBy { it.playbackViews }
3 -> StateSubscriptions.instance.getSubscriptions().sortedByDescending { it.playbackViews }
0 -> StateSubscriptions.instance.getSubscriptions().sortedBy({ u -> u.channel.name.lowercase() })
1 -> StateSubscriptions.instance.getSubscriptions().sortedByDescending({ u -> u.channel.name.lowercase() })
2 -> StateSubscriptions.instance.getSubscriptions().sortedBy { it.playbackViews * VIEW_PRIORITY + it.playbackSeconds }
3 -> StateSubscriptions.instance.getSubscriptions().sortedByDescending { it.playbackViews * VIEW_PRIORITY + it.playbackSeconds }
4 -> StateSubscriptions.instance.getSubscriptions().sortedBy { it.playbackSeconds }
5 -> StateSubscriptions.instance.getSubscriptions().sortedByDescending { it.playbackSeconds }
else -> throw IllegalStateException("Invalid sorting algorithm selected.");
@@ -65,4 +65,9 @@ class SubscriptionAdapter : RecyclerView.Adapter<SubscriptionViewHolder> {
notifyDataSetChanged();
}
companion object {
val VIEW_PRIORITY = 36000 * 3;
}
}
@@ -21,10 +21,13 @@ class SourceHeaderView : LinearLayout {
private val _sourceDescription: TextView;
private val _sourceVersion: TextView;
private val _sourcePlatformUrl: TextView;
private val _sourceRepositoryUrl: TextView;
private val _sourceScriptUrl: TextView;
private val _sourceSignature: TextView;
private val _sourcePlatformUrlContainer: LinearLayout;
private var _config : SourcePluginConfig? = null;
constructor(context: Context, attrs: AttributeSet? = null) : super(context, attrs) {
@@ -38,6 +41,8 @@ class SourceHeaderView : LinearLayout {
_sourceVersion = findViewById(R.id.source_version);
_sourceRepositoryUrl = findViewById(R.id.source_repo);
_sourcePlatformUrl = findViewById(R.id.source_platform);
_sourcePlatformUrlContainer = findViewById(R.id.source_platform_container);
_sourceScriptUrl = findViewById(R.id.source_script);
_sourceSignature = findViewById(R.id.source_signature);
@@ -53,6 +58,10 @@ class SourceHeaderView : LinearLayout {
if(!_config?.absoluteScriptUrl.isNullOrEmpty())
context.startActivity(Intent(Intent.ACTION_VIEW, Uri.parse(_config?.absoluteScriptUrl)));
};
_sourcePlatformUrl.setOnClickListener {
if(!_config?.platformUrl.isNullOrEmpty())
context.startActivity(Intent(Intent.ACTION_VIEW, Uri.parse(_config?.platformUrl)));
};
}
fun loadConfig(config: SourcePluginConfig, script: String?) {
@@ -74,6 +83,12 @@ class SourceHeaderView : LinearLayout {
_sourceRepositoryUrl.text = config.repositoryUrl;
_sourceAuthorID.text = "";
_sourcePlatformUrl.text = config.platformUrl ?: "";
if(!config.platformUrl.isNullOrEmpty())
_sourcePlatformUrlContainer.visibility = VISIBLE;
else
_sourcePlatformUrlContainer.visibility = GONE;
if(!config.authorUrl.isNullOrEmpty())
_sourceBy.setTextColor(resources.getColor(R.color.colorPrimary));
else
@@ -105,5 +120,7 @@ class SourceHeaderView : LinearLayout {
_sourceScriptUrl.text = "";
_sourceRepositoryUrl.text = "";
_sourceAuthorID.text = "";
_sourcePlatformUrl.text = "";
_sourcePlatformUrlContainer.visibility = GONE;
}
}
@@ -25,7 +25,7 @@ class SubscriptionBar : LinearLayout {
constructor(context: Context, attrs: AttributeSet? = null) : super(context, attrs) {
inflate(context, R.layout.view_subscription_bar, this);
val subscriptions = StateSubscriptions.instance.getSubscriptions().sortedByDescending { it.playbackViews };
val subscriptions = StateSubscriptions.instance.getSubscriptions().sortedByDescending { it.playbackSeconds };
_adapterView = findViewById<RecyclerView>(R.id.recycler_creators).asAny(subscriptions, orientation = RecyclerView.HORIZONTAL) {
it.onClick.subscribe { c ->
onClickChannel.emit(c.channel);
@@ -100,6 +100,30 @@
tools:text="3" />
</LinearLayout>
<!--Platform Url-->
<LinearLayout
android:id="@+id/source_platform_container"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:orientation="vertical">
<TextView
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:textSize="14dp"
android:textColor="@color/white"
android:layout_marginTop="10dp"
android:fontFamily="@font/inter_light"
android:text="@string/platform_url" />
<TextView
android:id="@+id/source_platform"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:textSize="14dp"
android:textColor="@color/colorPrimary"
android:fontFamily="@font/inter_extra_light"
tools:text="https://some.platform.url" />
</LinearLayout>
<!--Repo Url-->
<LinearLayout
android:layout_width="match_parent"
+5
View File
@@ -98,6 +98,7 @@
<string name="are_you_sure_delete_historical">Are you sure you want to remove these historical entries?</string>
<string name="removed">removed</string>
<string name="add_source">Add Source</string>
<string name="platform_url">Platform URL</string>
<string name="repository_url">Repository URL</string>
<string name="script_url">Script URL</string>
<string name="source_permissions_explanation">These are the permissions the plugin requires to function</string>
@@ -396,6 +397,10 @@
<string name="various_tests_against_a_custom_source">Various tests against a custom source</string>
<string name="writes_to_disk_till_no_space_is_left">Writes to disk till no space is left</string>
<string name="visibility">Visibility</string>
<string name="ratelimit">Rate-limit</string>
<string name="ratelimit_description">Settings related to rate-limiting this plugin\'s behavior</string>
<string name="ratelimit_sub_setting">Rate-limit Subscriptions</string>
<string name="ratelimit_sub_setting_description">Limit the amount of subscription requests made</string>
<string name="enable_where_this_plugins_content_are_visible">Enable where this plugin\'s content are visible</string>
<string name="show_content_in_home_tab">Show content in home tab</string>
<string name="show_content_in_search_results">Show content in search results</string>