Subscriptions ratelimit and warnings, Nebula login requirement, Subscription fetch setting, -1 sub hide

.gitlab-ci.yml

@@ -4,6 +4,7 @@ variables:
 stages:
 - buildAndDeployApkUnstable
 - buildAndDeployApkStable
+- buildAndDeployPlaystore
 
 buildAndDeployApkUnstable:
   stage: buildAndDeployApkUnstable
@@ -25,8 +26,8 @@ buildAndDeployApkStable:
     - branches
   when: manual
 
-buildAndDeployApkStable:
-  stage: buildAndDeployApkStable
+buildAndDeployPlaystore:
+  stage: buildAndDeployPlaystore
   script:
     - sh deploy-playstore.sh
   only:

app/build.gradle

@@ -95,7 +95,7 @@ android {
     }
 
     defaultConfig {
-        minSdk 29
+        minSdk 28
         targetSdk 33
         versionCode gitVersionCode
         versionName gitVersionName

app/src/main/AndroidManifest.xml

@@ -127,6 +127,10 @@
             android:name=".activities.ExceptionActivity"
             android:screenOrientation="portrait"
             android:theme="@style/Theme.FutoVideo.NoActionBar" />
+        <activity
+            android:name=".activities.CaptchaActivity"
+            android:screenOrientation="portrait"
+            android:theme="@style/Theme.FutoVideo.NoActionBar" />
         <activity
             android:name=".activities.LoginActivity"
             android:screenOrientation="portrait"

app/src/main/assets/scripts/source.js

@@ -64,6 +64,14 @@ class ScriptException extends Error {
         }
     }
 }
+class CaptchaRequiredException extends Error {
+    constructor(url, body) {
+        super(JSON.stringify({ 'plugin_type': 'CaptchaRequiredException', url, body }));
+        this.plugin_type = "CaptchaRequiredException";
+        this.url = url;
+        this.body = body;
+    }
+}
 class UnavailableException extends ScriptException {
     constructor(msg) {
         super("UnavailableException", msg);

app/src/main/java/com/futo/platformplayer/Settings.kt

@@ -4,6 +4,7 @@ import android.content.ActivityNotFoundException
 import android.content.Context
 import android.content.Intent
 import android.net.Uri
+import android.os.Build
 import android.webkit.CookieManager
 import androidx.lifecycle.lifecycleScope
 import com.futo.platformplayer.activities.*
@@ -63,7 +64,8 @@ class Settings : FragmentedStorageFileJson() {
         try {
             val i = Intent(Intent.ACTION_VIEW);
             val subject = "Feedback Grayjay";
-            val body = "Hey,\n\nI have some feedback on the Grayjay app.\nVersion information (version_name = ${BuildConfig.VERSION_NAME}, version_code = ${BuildConfig.VERSION_CODE}, flavor = ${BuildConfig.FLAVOR}, build_type = ${BuildConfig.BUILD_TYPE}})\n\n";
+            val body = "Hey,\n\nI have some feedback on the Grayjay app.\nVersion information (version_name = ${BuildConfig.VERSION_NAME}, version_code = ${BuildConfig.VERSION_CODE}, flavor = ${BuildConfig.FLAVOR}, build_type = ${BuildConfig.BUILD_TYPE}})\n" +
+                    "Device information (brand= ${Build.BRAND}, manufacturer = ${Build.MANUFACTURER}, device = ${Build.DEVICE}, version-sdk = ${Build.VERSION.SDK_INT}, version-os = ${Build.VERSION.BASE_OS})\n\n";
             val data = Uri.parse("mailto:grayjay@futo.org?subject=" + Uri.encode(subject) + "&body=" + Uri.encode(body));
             i.data = data;
 
@@ -140,7 +142,11 @@ class Settings : FragmentedStorageFileJson() {
                 return FeedStyle.THUMBNAIL;
         }
 
-        @FormField("Background Update", FieldForm.DROPDOWN, "Experimental background update for subscriptions cache (requires restart)", 6)
+        @FormField("Fetch on app boot", FieldForm.TOGGLE, "Shortly after opening the app, start fetching subscriptions.", 6)
+        @Serializable(with = FlexibleBooleanSerializer::class)
+        var fetchOnAppBoot: Boolean = true;
+
+        @FormField("Background Update", FieldForm.DROPDOWN, "Experimental background update for subscriptions cache (requires restart)", 7)
         @DropdownFieldOptionsId(R.array.background_interval)
         var subscriptionsBackgroundUpdateInterval: Int = 0;
 
@@ -156,7 +162,7 @@ class Settings : FragmentedStorageFileJson() {
         };
 
 
-        @FormField("Subscription Concurrency", FieldForm.DROPDOWN, "Specify how many threads are used to fetch channels (requires restart)", 7)
+        @FormField("Subscription Concurrency", FieldForm.DROPDOWN, "Specify how many threads are used to fetch channels (requires restart)", 8)
         @DropdownFieldOptionsId(R.array.thread_count)
         var subscriptionConcurrency: Int = 3;
 
@@ -213,7 +219,7 @@ class Settings : FragmentedStorageFileJson() {
 
         fun isAutoRotate() = autoRotate == 1 || (autoRotate == 2 && StateApp.instance.getCurrentSystemAutoRotate());
 
-        @FormField("Auto-Rotate Dead Zone", FieldForm.DROPDOWN, "Auto-rotate deadzone in degrees", 5)
+        @FormField("Auto-Rotate Dead Zone", FieldForm.DROPDOWN, "This prevents the device from rotating within the given amount of degrees.", 5)
         @DropdownFieldOptionsId(R.array.auto_rotate_dead_zone)
         var autoRotateDeadZone: Int = 0;
 

app/src/main/java/com/futo/platformplayer/activities/AddSourceActivity.kt

@@ -75,10 +75,10 @@ class AddSourceActivity : AppCompatActivity() {
         _buttonInstall = findViewById(R.id.button_install);
 
         _buttonBack.setOnClickListener {
-            onBackPressed();
+            finish();
         };
         _buttonCancel.setOnClickListener {
-            onBackPressed();
+            finish();
         }
         _buttonInstall.setOnClickListener {
             _config?.let {

app/src/main/java/com/futo/platformplayer/activities/AddSourceOptionsActivity.kt

@@ -1,7 +1,10 @@
 package com.futo.platformplayer.activities
 
+import android.content.Intent
+import android.net.Uri
 import android.os.Bundle
 import android.widget.*
+import androidx.activity.result.contract.ActivityResultContracts
 import androidx.appcompat.app.AppCompatActivity
 import com.futo.platformplayer.*
 import com.futo.platformplayer.views.buttons.BigButton
@@ -14,6 +17,31 @@ class AddSourceOptionsActivity : AppCompatActivity() {
     lateinit var _buttonQR: BigButton;
     lateinit var _buttonURL: BigButton;
 
+    private val _qrCodeResultLauncher = registerForActivityResult(ActivityResultContracts.StartActivityForResult()) { result ->
+        val scanResult = IntentIntegrator.parseActivityResult(result.resultCode, result.data)
+        scanResult?.let {
+            val content = it.contents
+            if (content == null) {
+                UIDialogs.toast(this, "Failed to scan QR code")
+                return@let
+            }
+
+            val url = if (content.startsWith("https://")) {
+                content
+            } else if (content.startsWith("grayjay://plugin/")) {
+                content.substring("grayjay://plugin/".length)
+            } else {
+                UIDialogs.toast(this, "Not a plugin URL")
+                return@let;
+            }
+
+            val intent = Intent(this, AddSourceActivity::class.java).apply {
+                data = Uri.parse(url);
+            };
+            startActivity(intent);
+        }
+    }
+
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState);
         setContentView(R.layout.activity_add_source_options);
@@ -37,8 +65,9 @@ class AddSourceOptionsActivity : AppCompatActivity() {
             integrator.setBeepEnabled(false)
             integrator.setBarcodeImageEnabled(true)
             integrator.setCaptureActivity(QRCaptureActivity::class.java);
-            integrator.initiateScan()
+            _qrCodeResultLauncher.launch(integrator.createScanIntent())
         }
+
         _buttonURL.onClick.subscribe {
             UIDialogs.toast(this, "Not implemented yet..");
         }

app/src/main/java/com/futo/platformplayer/activities/CaptchaActivity.kt

@@ -0,0 +1,120 @@
+package com.futo.platformplayer.activities
+
+import android.content.Context
+import android.content.Intent
+import android.os.Bundle
+import android.webkit.CookieManager
+import android.webkit.WebView
+import android.widget.Button
+import androidx.appcompat.app.AppCompatActivity
+import androidx.lifecycle.lifecycleScope
+import com.futo.platformplayer.*
+import com.futo.platformplayer.api.media.platforms.js.SourceAuth
+import com.futo.platformplayer.api.media.platforms.js.SourceCaptchaData
+import com.futo.platformplayer.api.media.platforms.js.SourcePluginAuthConfig
+import com.futo.platformplayer.api.media.platforms.js.SourcePluginCaptchaConfig
+import com.futo.platformplayer.api.media.platforms.js.SourcePluginConfig
+import com.futo.platformplayer.engine.IV8PluginConfig
+import com.futo.platformplayer.logging.Logger
+import com.futo.platformplayer.others.CaptchaWebViewClient
+import com.futo.platformplayer.others.LoginWebViewClient
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.encodeToString
+import kotlinx.serialization.json.Json
+import java.io.File
+import java.lang.Exception
+import java.util.UUID
+
+class CaptchaActivity : AppCompatActivity() {
+    private lateinit var _webView: WebView;
+    private lateinit var _buttonClose: Button;
+
+    override fun onCreate(savedInstanceState: Bundle?) {
+        super.onCreate(savedInstanceState);
+        setContentView(R.layout.activity_captcha);
+        setNavigationBarColorAndIcons();
+
+        _buttonClose = findViewById(R.id.button_close);
+        _buttonClose.setOnClickListener { finish(); };
+
+        _webView = findViewById(R.id.web_view);
+        _webView.settings.javaScriptEnabled = true;
+        CookieManager.getInstance().setAcceptCookie(true);
+
+
+        val config = if(intent.hasExtra("plugin"))
+            Json.decodeFromString<SourcePluginConfig>(intent.getStringExtra("plugin")!!);
+        else null;
+
+        val captchaConfig = if(config != null)
+            config.captcha ?: throw IllegalStateException("Plugin has no captcha support");
+        else if(intent.hasExtra("captcha"))
+            Json.decodeFromString<SourcePluginCaptchaConfig>(intent.getStringExtra("captcha")!!);
+        else throw IllegalStateException("No valid configuration?");
+        //TODO: Backwards compat removal?
+
+        val extraUrl = if (intent.hasExtra("url"))
+            intent.getStringExtra("url");
+        else null;
+
+        val extraBody = if (intent.hasExtra("body"))
+            intent.getStringExtra("body");
+        else null;
+
+        _webView.settings.userAgentString = captchaConfig.userAgent ?: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36";
+        _webView.settings.useWideViewPort = true;
+        _webView.settings.loadWithOverviewMode = true;
+
+        val webViewClient = if(config != null) CaptchaWebViewClient(config) else CaptchaWebViewClient(captchaConfig);
+        webViewClient.onCaptchaFinished.subscribe { captcha ->
+            _callback?.let {
+                _callback = null;
+                it.invoke(captcha);
+            }
+            finish();
+        };
+        _webView.settings.domStorageEnabled = true;
+        _webView.webViewClient = webViewClient;
+
+        if(captchaConfig.captchaUrl != null)
+            _webView.loadUrl(captchaConfig.captchaUrl);
+        else if(extraUrl != null && extraBody != null)
+            _webView.loadDataWithBaseURL(extraUrl, extraBody, "text/html", "utf-8", null);
+        else if(extraUrl != null)
+            _webView.loadUrl(extraUrl);
+        else throw IllegalStateException("No valid captcha info provided");
+    }
+
+    override fun finish() {
+        lifecycleScope.launch(Dispatchers.Main) {
+            _webView.loadUrl("about:blank");
+        }
+        _callback?.let {
+            _callback = null;
+            it.invoke(null);
+        }
+        super.finish();
+    }
+
+    companion object {
+        private val TAG = "CaptchaActivity";
+        private var _callback: ((SourceCaptchaData?) -> Unit)? = null;
+
+        private fun getCaptchaIntent(context: Context, config: SourcePluginConfig, url: String? = null, body: String? = null): Intent {
+            val intent = Intent(context, CaptchaActivity::class.java);
+            if(url != null)
+                intent.putExtra("url", url);
+            if(body != null)
+                intent.putExtra("body", body);
+            intent.putExtra("plugin", Json.encodeToString(config));
+            return intent;
+        }
+
+        fun showCaptcha(context: Context, config: SourcePluginConfig, url: String? = null, body: String? = null, callback: ((SourceCaptchaData?) -> Unit)? = null) {
+            _callback = callback;
+            context.startActivity(getCaptchaIntent(context, config, url, body));
+        }
+    }
+}

app/src/main/java/com/futo/platformplayer/activities/ExceptionActivity.kt

@@ -1,6 +1,7 @@
 package com.futo.platformplayer.activities
 
 import android.content.Intent
+import android.os.Build
 import android.os.Bundle
 import android.widget.LinearLayout
 import android.widget.TextView
@@ -40,7 +41,8 @@ class ExceptionActivity : AppCompatActivity() {
         val context = intent.getStringExtra(EXTRA_CONTEXT) ?: "Unknown Context";
         val stack = intent.getStringExtra(EXTRA_STACK) ?: "Something went wrong... missing stack trace?";
 
-        val exceptionString = "Version information (version_name = ${BuildConfig.VERSION_NAME}, version_code = ${BuildConfig.VERSION_CODE}, flavor = ${BuildConfig.FLAVOR}, build_type = ${BuildConfig.BUILD_TYPE})\n\n" +
+        val exceptionString = "Version information (version_name = ${BuildConfig.VERSION_NAME}, version_code = ${BuildConfig.VERSION_CODE}, flavor = ${BuildConfig.FLAVOR}, build_type = ${BuildConfig.BUILD_TYPE})\n" +
+                "Device information (brand= ${Build.BRAND}, manufacturer = ${Build.MANUFACTURER}, device = ${Build.DEVICE}, version-sdk = ${Build.VERSION.SDK_INT}, version-os = ${Build.VERSION.BASE_OS})\n\n" +
                 Logging.buildLogString(LogLevel.ERROR, TAG, "Uncaught exception (\"$context\"): $stack");
         try {
             val file = File(filesDir, "log.txt");

app/src/main/java/com/futo/platformplayer/activities/LoginActivity.kt

@@ -3,7 +3,9 @@ package com.futo.platformplayer.activities
 import android.content.Context
 import android.content.Intent
 import android.os.Bundle
+import android.webkit.ConsoleMessage
 import android.webkit.CookieManager
+import android.webkit.WebChromeClient
 import android.webkit.WebView
 import androidx.appcompat.app.AppCompatActivity
 import androidx.lifecycle.lifecycleScope
@@ -68,9 +70,15 @@ class LoginActivity : AppCompatActivity() {
                 view?.evaluateJavascript("setTimeout(()=> document.querySelector(\"${authConfig.loginButton}\")?.click(), 1000)", {});
             }
         }
-        //TODO: Required for some...TBD what to do with it. Clear on finish?
         _webView.settings.domStorageEnabled = true;
 
+        /*
+        _webView.webChromeClient = object: WebChromeClient() {
+            override fun onConsoleMessage(consoleMessage: ConsoleMessage?): Boolean {
+                Logger.w(TAG, "Login Console: " + consoleMessage?.message());
+                return super.onConsoleMessage(consoleMessage);
+            }
+        }*/
         _webView.webViewClient = webViewClient;
         _webView.loadUrl(authConfig.loginUrl);
     }

app/src/main/java/com/futo/platformplayer/activities/MainActivity.kt

@@ -607,6 +607,7 @@ class MainActivity : AppCompatActivity, IWithResultLauncher {
                 return;
             };
         };
+
         val name = when(type) {
             "Playlist" -> recon.split("\n").filter { !it.startsWith(ManagedStore.RECONSTRUCTION_HEADER_OPERATOR) }.firstOrNull() ?: type;
             else -> type
@@ -894,7 +895,7 @@ class MainActivity : AppCompatActivity, IWithResultLauncher {
 
 
 
-
+    //TODO: Only calls last handler due to missing request codes on ActivityResultLaunchers.
     private var resultLauncherMap =  mutableMapOf<Int, (ActivityResult)->Unit>();
     private var requestCode: Int? = -1;
     private val resultLauncher: ActivityResultLauncher<Intent> = registerForActivityResult(

app/src/main/java/com/futo/platformplayer/activities/PolycentricImportProfileActivity.kt

@@ -5,6 +5,7 @@ import android.os.Bundle
 import android.widget.EditText
 import android.widget.ImageButton
 import android.widget.LinearLayout
+import androidx.activity.result.contract.ActivityResultContracts
 import androidx.appcompat.app.AppCompatActivity
 import androidx.lifecycle.lifecycleScope
 import com.futo.platformplayer.R
@@ -14,6 +15,7 @@ import com.futo.platformplayer.setNavigationBarColorAndIcons
 import com.futo.platformplayer.states.StatePolycentric
 import com.futo.polycentric.core.*
 import com.google.zxing.integration.android.IntentIntegrator
+import com.journeyapps.barcodescanner.CaptureActivity
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.GlobalScope
 import kotlinx.coroutines.launch
@@ -27,6 +29,16 @@ class PolycentricImportProfileActivity : AppCompatActivity() {
     private lateinit var _buttonImportProfile: LinearLayout;
     private lateinit var _editProfile: EditText;
 
+    private val _qrCodeResultLauncher = registerForActivityResult(ActivityResultContracts.StartActivityForResult()) { result ->
+        val scanResult = IntentIntegrator.parseActivityResult(result.resultCode, result.data)
+        scanResult?.let {
+            if (it.contents != null) {
+                val scannedUrl = it.contents
+                import(scannedUrl)
+            }
+        }
+    }
+
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState);
         setContentView(R.layout.activity_polycentric_import_profile);
@@ -45,10 +57,15 @@ class PolycentricImportProfileActivity : AppCompatActivity() {
         };
 
         _buttonScanProfile.setOnClickListener {
-            val integrator = IntentIntegrator(this);
-            integrator.setDesiredBarcodeFormats(IntentIntegrator.QR_CODE);
-            integrator.setPrompt("Scan a QR code");
-            integrator.initiateScan();
+            val integrator = IntentIntegrator(this)
+            integrator.setDesiredBarcodeFormats(IntentIntegrator.QR_CODE)
+            integrator.setPrompt("Scan a QR code")
+            integrator.setOrientationLocked(true);
+            integrator.setCameraId(0)
+            integrator.setBeepEnabled(false)
+            integrator.setBarcodeImageEnabled(true)
+            integrator.setCaptureActivity(QRCaptureActivity::class.java);
+            _qrCodeResultLauncher.launch(integrator.createScanIntent())
         };
 
         _buttonImportProfile.setOnClickListener {
@@ -66,18 +83,6 @@ class PolycentricImportProfileActivity : AppCompatActivity() {
         }
     }
 
-    override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) {
-        val result = IntentIntegrator.parseActivityResult(requestCode, resultCode, data)
-        if (result != null) {
-            if (result.contents != null) {
-                val scannedUrl = result.contents;
-                import(scannedUrl);
-            }
-        } else {
-            super.onActivityResult(requestCode, resultCode, data)
-        }
-    }
-
     private fun import(url: String) {
         if (!url.startsWith("polycentric://")) {
             UIDialogs.toast(this, "Not a valid URL");
@@ -126,4 +131,8 @@ class PolycentricImportProfileActivity : AppCompatActivity() {
     companion object {
         private const val TAG = "PolycentricImportProfileActivity";
     }
+
+    class QRCaptureActivity: CaptureActivity() {
+
+    }
 }

app/src/main/java/com/futo/platformplayer/activities/SettingsActivity.kt

@@ -52,17 +52,6 @@ class SettingsActivity : AppCompatActivity(), IWithResultLauncher {
             startActivity(Intent(this, DeveloperActivity::class.java));
         }
 
-        var devCounter = 0;
-        _form.findField("code")?.assume<ReadOnlyTextField>()?.setOnClickListener {
-            devCounter++;
-            if(devCounter > 5) {
-                devCounter = 0;
-                SettingsDev.instance.developerMode = true;
-                SettingsDev.instance.save();
-                updateDevMode();
-                UIDialogs.toast(this, "You are now in developer mode");
-            }
-        };
         _lastActivity = this;
 
         reloadSettings();
@@ -72,6 +61,18 @@ class SettingsActivity : AppCompatActivity(), IWithResultLauncher {
         _loader.start();
         _form.fromObject(lifecycleScope, Settings.instance) {
             _loader.stop();
+
+            var devCounter = 0;
+            _form.findField("code")?.assume<ReadOnlyTextField>()?.setOnClickListener {
+                devCounter++;
+                if(devCounter > 5) {
+                    devCounter = 0;
+                    SettingsDev.instance.developerMode = true;
+                    SettingsDev.instance.save();
+                    updateDevMode();
+                    UIDialogs.toast(this, "You are now in developer mode");
+                }
+            };
         };
     }
 

app/src/main/java/com/futo/platformplayer/api/media/PlatformClientPool.kt

@@ -6,6 +6,7 @@ import com.futo.platformplayer.api.media.platforms.js.JSClient
 import com.futo.platformplayer.constructs.Event1
 import com.futo.platformplayer.constructs.Event2
 import com.futo.platformplayer.logging.Logger
+import com.futo.platformplayer.states.StateApp
 
 class PlatformClientPool {
     private val _parent: JSClient;
@@ -51,6 +52,11 @@ class PlatformClientPool {
             if(reserved == null && _pool.size < capacity) {
                 Logger.i(TAG, "Started additional [${_parent.name}] client in pool [${_poolName}] (${_pool.size + 1}/${capacity})");
                 reserved = _parent.getCopy();
+
+                reserved?.onCaptchaException?.subscribe { client, ex ->
+                    StateApp.instance.handleCaptchaException(client, ex);
+                };
+
                 reserved?.initialize();
                 _pool[reserved!!] = _poolCounter;
             }

app/src/main/java/com/futo/platformplayer/api/media/models/ResultCapabilities.kt

@@ -27,6 +27,7 @@ class ResultCapabilities(
         const val TYPE_VIDEOS = "VIDEOS";
         const val TYPE_STREAMS = "STREAMS";
         const val TYPE_LIVE = "LIVE";
+        const val TYPE_POSTS = "POSTS";
         const val TYPE_MIXED = "MIXED";
 
         const val ORDER_CHONOLOGICAL = "CHRONOLOGICAL";

app/src/main/java/com/futo/platformplayer/api/media/models/comments/PolycentricPlatformComment.kt

@@ -39,4 +39,8 @@ class PolycentricPlatformComment : IPlatformComment {
     fun cloneWithUpdatedReplyCount(replyCount: Int?): PolycentricPlatformComment {
         return PolycentricPlatformComment(contextUrl, author, message, rating, date, reference, replyCount);
     }
+
+    companion object {
+        val MAX_COMMENT_SIZE = 2000
+    }
 }

app/src/main/java/com/futo/platformplayer/api/media/models/video/SerializedPlatformNestedContent.kt

@@ -30,7 +30,7 @@ open class SerializedPlatformNestedContent(
     override val contentProvider: String?,
     override val contentThumbnails: Thumbnails
 ) : IPlatformNestedContent, SerializedPlatformContent {
-    final override val contentType: ContentType get() = ContentType.MEDIA;
+    final override val contentType: ContentType get() = ContentType.NESTED_VIDEO;
 
     override val contentPlugin: String? = StatePlatform.instance.getContentClientOrNull(contentUrl)?.id;
     override val contentSupported: Boolean get() = contentPlugin != null;

app/src/main/java/com/futo/platformplayer/api/media/platforms/js/DevJSClient.kt

@@ -15,29 +15,36 @@ class DevJSClient : JSClient {
 
     private val _devScript: String;
     private var _auth: SourceAuth? = null;
+    private var _captcha: SourceCaptchaData? = null;
 
     val devID: String;
 
-    constructor(context: Context, config: SourcePluginConfig, script: String, auth: SourceAuth? = null, devID: String? = null): super(context, SourcePluginDescriptor(config, auth?.toEncrypted(), listOf("DEV")), null, script) {
+    constructor(context: Context, config: SourcePluginConfig, script: String, auth: SourceAuth? = null, captcha: SourceCaptchaData? = null, devID: String? = null): super(context, SourcePluginDescriptor(config, auth?.toEncrypted(), captcha?.toEncrypted(), listOf("DEV")), null, script) {
         _devScript = script;
         _auth = auth;
+        _captcha = captcha;
         this.devID = devID ?: UUID.randomUUID().toString().substring(0, 5);
     }
-    constructor(context: Context, descriptor: SourcePluginDescriptor, script: String, auth: SourceAuth? = null, savedState: String? = null, devID: String? = null): super(context, descriptor, savedState, script) {
+    //TODO: Misisng auth/captcha pass on purpose?
+    constructor(context: Context, descriptor: SourcePluginDescriptor, script: String, auth: SourceAuth? = null, captcha: SourceCaptchaData? = null, savedState: String? = null, devID: String? = null): super(context, descriptor, savedState, script) {
         _devScript = script;
         _auth = auth;
+        _captcha = captcha;
         this.devID = devID ?: UUID.randomUUID().toString().substring(0, 5);
     }
 
+    fun setCaptcha(captcha: SourceCaptchaData? = null) {
+        _captcha = captcha;
+    }
     fun setAuth(auth: SourceAuth? = null) {
         _auth = auth;
     }
     fun recreate(context: Context): DevJSClient {
-        return DevJSClient(context, config, _devScript, _auth, devID);
+        return DevJSClient(context, config, _devScript, _auth, _captcha, devID);
     }
 
     override fun getCopy(): JSClient {
-        return DevJSClient(_context, descriptor, _script, _auth, saveState(), devID);
+        return DevJSClient(_context, descriptor, _script, _auth, _captcha, saveState(), devID);
     }
 
     override fun initialize() {

app/src/main/java/com/futo/platformplayer/api/media/platforms/js/JSClient.kt

@@ -25,9 +25,11 @@ import com.futo.platformplayer.api.media.platforms.js.internal.*
 import com.futo.platformplayer.api.media.platforms.js.models.*
 import com.futo.platformplayer.api.media.structures.IPager
 import com.futo.platformplayer.constructs.Event1
+import com.futo.platformplayer.constructs.Event2
 import com.futo.platformplayer.engine.V8Plugin
 import com.futo.platformplayer.engine.exceptions.PluginEngineException
 import com.futo.platformplayer.engine.exceptions.PluginEngineStoppedException
+import com.futo.platformplayer.engine.exceptions.ScriptCaptchaRequiredException
 import com.futo.platformplayer.engine.exceptions.ScriptImplementationException
 import com.futo.platformplayer.engine.exceptions.ScriptValidationException
 import com.futo.platformplayer.logging.Logger
@@ -61,6 +63,7 @@ open class JSClient : IPlatformClient {
     private var _enabled: Boolean = false;
 
     private val _auth: SourceAuth?;
+    private val _captcha: SourceCaptchaData?;
 
     private val _injectedSaveState: String?;
 
@@ -87,6 +90,7 @@ open class JSClient : IPlatformClient {
     val enableInHome get() = descriptor.appSettings.tabEnabled.enableHome ?: true
 
     val onDisabled = Event1<JSClient>();
+    val onCaptchaException = Event2<JSClient, ScriptCaptchaRequiredException>();
 
     constructor(context: Context, descriptor: SourcePluginDescriptor, saveState: String? = null) {
         this._context = context;
@@ -95,10 +99,11 @@ open class JSClient : IPlatformClient {
         this.descriptor = descriptor;
         _injectedSaveState = saveState;
         _auth = descriptor.getAuth();
+        _captcha = descriptor.getCaptchaData();
         flags = descriptor.flags.toTypedArray();
 
-        _client = JSHttpClient(this);
-        _clientAuth = JSHttpClient(this, _auth);
+        _client = JSHttpClient(this, null, _captcha);
+        _clientAuth = JSHttpClient(this, _auth, _captcha);
         _plugin = V8Plugin(context, descriptor.config, null, _client, _clientAuth);
         _plugin.withDependency(context, "scripts/polyfil.js");
         _plugin.withDependency(context, "scripts/source.js");
@@ -110,6 +115,11 @@ open class JSClient : IPlatformClient {
         }
         else
             throw IllegalStateException("Script for plugin [${descriptor.config.name}] was not available");
+
+        _plugin.onScriptException.subscribe {
+            if(it is ScriptCaptchaRequiredException)
+                onCaptchaException.emit(this, it);
+        };
     }
     constructor(context: Context, descriptor: SourcePluginDescriptor, saveState: String?, script: String) {
         this._context = context;
@@ -118,15 +128,21 @@ open class JSClient : IPlatformClient {
         this.descriptor = descriptor;
         _injectedSaveState = saveState;
         _auth = descriptor.getAuth();
+        _captcha = descriptor.getCaptchaData();
         flags = descriptor.flags.toTypedArray();
 
-        _client = JSHttpClient(this);
-        _clientAuth = JSHttpClient(this, _auth);
+        _client = JSHttpClient(this, null, _captcha);
+        _clientAuth = JSHttpClient(this, _auth, _captcha);
         _plugin = V8Plugin(context, descriptor.config, script, _client, _clientAuth);
         _plugin.withDependency(context, "scripts/polyfil.js");
         _plugin.withDependency(context, "scripts/source.js");
         _plugin.withScript(script);
         _script = script;
+
+        _plugin.onScriptException.subscribe {
+            if(it is ScriptCaptchaRequiredException)
+                onCaptchaException.emit(this, it);
+        };
     }
 
     open fun getCopy(): JSClient {

app/src/main/java/com/futo/platformplayer/api/media/platforms/js/SourceCaptchaData.kt

@@ -0,0 +1,49 @@
+package com.futo.platformplayer.api.media.platforms.js
+
+import com.futo.platformplayer.encryption.EncryptionProvider
+import com.futo.platformplayer.logging.Logger
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.encodeToString
+import kotlinx.serialization.json.Json
+
+data class SourceCaptchaData(val cookieMap: HashMap<String, HashMap<String, String>>? = null, val headers: Map<String, Map<String, String>> = mapOf()) {
+    override fun toString(): String {
+        return "(headers: '$headers', cookieString: '$cookieMap')";
+    }
+
+    fun toEncrypted(): String{
+        return EncryptionProvider.instance.encrypt(serialize());
+    }
+
+    private fun serialize(): String {
+        return Json.encodeToString(SerializedCaptchaData(cookieMap, headers));
+    }
+
+    companion object {
+        val TAG = "SourceAuth";
+
+        fun fromEncrypted(encrypted: String?): SourceCaptchaData? {
+            if(encrypted == null)
+                return null;
+
+            val decrypted = EncryptionProvider.instance.decrypt(encrypted);
+            try {
+                return deserialize(decrypted);
+            }
+            catch(ex: Throwable) {
+                Logger.e(TAG, "Failed to deserialize authentication", ex);
+                return null;
+            }
+        }
+
+        fun deserialize(str: String): SourceCaptchaData {
+            val data = Json.decodeFromString<SerializedCaptchaData>(str);
+            return SourceCaptchaData(data.cookieMap, data.headers);
+        }
+    }
+
+    @Serializable
+    data class SerializedCaptchaData(val cookieMap: HashMap<String, HashMap<String, String>>?,
+                              val headers: Map<String, Map<String, String>> = mapOf())
+}

app/src/main/java/com/futo/platformplayer/api/media/platforms/js/SourcePluginCaptchaConfig.kt

@@ -0,0 +1,12 @@
+package com.futo.platformplayer.api.media.platforms.js
+
+import kotlinx.serialization.Serializable
+
+@Serializable
+class SourcePluginCaptchaConfig(
+    val captchaUrl: String? = null,
+    val completionUrl: String? = null,
+    val cookiesToFind: List<String>? = null,
+    val userAgent: String? = null,
+    val cookiesExclOthers: Boolean = true
+)

app/src/main/java/com/futo/platformplayer/api/media/platforms/js/SourcePluginConfig.kt

@@ -35,11 +35,13 @@ class SourcePluginConfig(
 
     val settings: List<Setting> = listOf(),
 
+    var captcha: SourcePluginCaptchaConfig? = null,
     val authentication: SourcePluginAuthConfig? = null,
     var sourceUrl: String? = null,
     val constants: HashMap<String, String> = hashMapOf(),
 
     //TODO: These should be vals...but prob for serialization reasons cannot be changed.
+    var subscriptionRateLimit: Int? = null,
     var enableInSearch: Boolean = true,
     var enableInHome: Boolean = true,
     var supportedClaimTypes: List<Int> = listOf()

app/src/main/java/com/futo/platformplayer/api/media/platforms/js/SourcePluginDescriptor.kt

@@ -13,22 +13,28 @@ class SourcePluginDescriptor {
 
     var appSettings: AppPluginSettings = AppPluginSettings();
 
-    var authEncrypted: String?
+    var authEncrypted: String? = null
+        private set;
+    var captchaEncrypted: String? = null
         private set;
 
     val flags: List<String>;
 
     @kotlinx.serialization.Transient
     val onAuthChanged = Event0();
+    @kotlinx.serialization.Transient
+    val onCaptchaChanged = Event0();
 
-    constructor(config :SourcePluginConfig, authEncrypted: String? = null) {
+    constructor(config :SourcePluginConfig, authEncrypted: String? = null, captchaEncrypted: String? = null) {
         this.config = config;
         this.authEncrypted = authEncrypted;
+        this.captchaEncrypted = captchaEncrypted;
         this.flags = listOf();
     }
-    constructor(config :SourcePluginConfig, authEncrypted: String? = null, flags: List<String>) {
+    constructor(config :SourcePluginConfig, authEncrypted: String? = null, captchaEncrypted: String? = null, flags: List<String>) {
         this.config = config;
         this.authEncrypted = authEncrypted;
+        this.captchaEncrypted = captchaEncrypted;
         this.flags = flags;
     }
 
@@ -41,6 +47,13 @@ class SourcePluginDescriptor {
         return map;
     }
 
+    fun updateCaptcha(captcha: SourceCaptchaData?) {
+        captchaEncrypted = captcha?.toEncrypted();
+        onCaptchaChanged.emit();
+    }
+    fun getCaptchaData(): SourceCaptchaData? {
+        return SourceCaptchaData.fromEncrypted(captchaEncrypted);
+    }
 
     fun updateAuth(str: SourceAuth?) {
         authEncrypted = str?.toEncrypted();

app/src/main/java/com/futo/platformplayer/api/media/platforms/js/internal/JSHttpClient.kt

@@ -5,65 +5,72 @@ import com.futo.platformplayer.logging.Logger
 import com.futo.platformplayer.api.http.ManagedHttpClient
 import com.futo.platformplayer.api.media.platforms.js.JSClient
 import com.futo.platformplayer.api.media.platforms.js.SourceAuth
+import com.futo.platformplayer.api.media.platforms.js.SourceCaptchaData
 import com.futo.platformplayer.matchesDomain
 
 class JSHttpClient : ManagedHttpClient {
     private val _jsClient: JSClient?;
     private val _auth: SourceAuth?;
+    private val _captcha: SourceCaptchaData?;
 
     var doUpdateCookies: Boolean = true;
     var doApplyCookies: Boolean = true;
     var doAllowNewCookies: Boolean = true;
     val isLoggedIn: Boolean get() = _auth != null;
 
-    private var _currentCookieMap: HashMap<String, HashMap<String, String>>?;
+    private var _currentCookieMap: HashMap<String, HashMap<String, String>>;
 
-    constructor(jsClient: JSClient?, auth: SourceAuth? = null) : super() {
+    constructor(jsClient: JSClient?, auth: SourceAuth? = null, captcha: SourceCaptchaData? = null) : super() {
         _jsClient = jsClient;
         _auth = auth;
+        _captcha = captcha;
 
+        _currentCookieMap = hashMapOf();
         if(!auth?.cookieMap.isNullOrEmpty()) {
-            _currentCookieMap = hashMapOf();
             for(domainCookies in auth!!.cookieMap!!)
-                _currentCookieMap!!.put(domainCookies.key, HashMap(domainCookies.value));
+                _currentCookieMap.put(domainCookies.key, HashMap(domainCookies.value));
         }
-        else _currentCookieMap = null;
+        if(!captcha?.cookieMap.isNullOrEmpty()) {
+            for(domainCookies in captcha!!.cookieMap!!)
+                _currentCookieMap.put(domainCookies.key, HashMap(domainCookies.value));
+        }
+
     }
 
     override fun clone(): ManagedHttpClient {
         val newClient = JSHttpClient(_jsClient, _auth);
         newClient._currentCookieMap = if(_currentCookieMap != null)
-            HashMap(_currentCookieMap!!.toList().associate { Pair(it.first, HashMap(it.second)) })
+            HashMap(_currentCookieMap.toList().associate { Pair(it.first, HashMap(it.second)) })
         else
-            null;
+            hashMapOf();
         return newClient;
     }
 
     override fun beforeRequest(request: Request) {
+        val domain = Uri.parse(request.url).host!!.lowercase();
+
         val auth = _auth;
         if (auth != null) {
-            val domain = Uri.parse(request.url).host!!.lowercase();
-
             //TODO: Possibly add doApplyHeaders
             for (header in auth.headers.filter { domain.matchesDomain(it.key) }.flatMap { it.value.entries })
                 request.headers[header.key] = header.value;
+        }
 
-            if(doApplyCookies) {
-                if (!_currentCookieMap.isNullOrEmpty()) {
-                    val cookiesToApply = hashMapOf<String, String>();
-                    synchronized(_currentCookieMap!!) {
-                        for(cookie in _currentCookieMap!!
-                                .filter { domain.matchesDomain(it.key) }
-                                .flatMap { it.value.toList() })
-                            cookiesToApply[cookie.first] = cookie.second;
-                    };
+        if(doApplyCookies) {
+            if (!_currentCookieMap.isNullOrEmpty()) {
+                val cookiesToApply = hashMapOf<String, String>();
+                synchronized(_currentCookieMap!!) {
+                    for(cookie in _currentCookieMap!!
+                        .filter { domain.matchesDomain(it.key) }
+                        .flatMap { it.value.toList() })
+                        cookiesToApply[cookie.first] = cookie.second;
+                };
 
-                    if(cookiesToApply.size > 0) {
-                        val cookieString = cookiesToApply.map { it.key + "=" + it.value }.joinToString("; ");
-                        request.headers["Cookie"] = cookieString;
-                    }
-                    //printTestCode(request.url, request.body, auth.headers, cookieString, request.headers.filter { !auth.headers.containsKey(it.key) });
+                if(cookiesToApply.size > 0) {
+                    val cookieString = cookiesToApply.map { it.key + "=" + it.value }.joinToString("; ");
+                    request.headers["Cookie"] = cookieString;
                 }
+                //printTestCode(request.url, request.body, auth.headers, cookieString, request.headers.filter { !auth.headers.containsKey(it.key) });
             }
         }
 
@@ -80,7 +87,7 @@ class JSHttpClient : ManagedHttpClient {
             val defaultCookieDomain =
                 "." + domainParts.drop(domainParts.size - 2).joinToString(".");
             for (header in resp.headers) {
-                if (_currentCookieMap != null && header.key.lowercase() == "set-cookie") {
+                if ((_auth != null || _currentCookieMap.isNotEmpty()) && header.key.lowercase() == "set-cookie") {
                     val newCookies = cookieStringToMap(header.value);
                     for (cookie in newCookies) {
                         val endIndex = cookie.value.indexOf(";");
@@ -155,4 +162,5 @@ class JSHttpClient : ManagedHttpClient {
 
         Logger.i("Testing", code);
     }
+
 }

app/src/main/java/com/futo/platformplayer/api/media/structures/DedupContentPager.kt

@@ -25,7 +25,8 @@ class DedupContentPager : IPager<IPlatformContent>, IAsyncPager<IPlatformContent
         _currentResults = dedupResults(_basePager.getResults());
     }
 
-    override fun hasMorePages(): Boolean = _basePager.hasMorePages();
+    override fun hasMorePages(): Boolean =
+        _basePager.hasMorePages();
     override fun nextPage() {
         _basePager.nextPage()
         _currentResults = dedupResults(_basePager.getResults());

app/src/main/java/com/futo/platformplayer/api/media/structures/MultiChronoContentPager.kt

@@ -7,7 +7,7 @@ import java.util.stream.IntStream
  * A PlatformContent MultiPager that orders the results of a page based on the datetime of a content item
  */
 class MultiChronoContentPager : MultiPager<IPlatformContent> {
-    constructor(pagers : Array<IPager<IPlatformContent>>, allowFailure: Boolean = false) : super(pagers.map { it }.toList(), allowFailure) {}
+    constructor(pagers : Array<IPager<IPlatformContent>>, allowFailure: Boolean = false, pageSize: Int = 9) : super(pagers.map { it }.toList(), allowFailure, pageSize) {}
 
     @Synchronized
     override fun selectItemIndex(options: Array<SelectionOption<IPlatformContent>>): Int {

app/src/main/java/com/futo/platformplayer/api/media/structures/MultiPager.kt

@@ -16,7 +16,7 @@ abstract class MultiPager<T> : IPager<T> {
     protected val _subSinglePagers : MutableList<SingleItemPager<T>>;
     protected val _failedPagers: ArrayList<IPager<T>> = arrayListOf();
 
-    private val _pageSize : Int = 9;
+    private var _pageSize : Int = 9;
 
     private var _didInitialize = false;
 
@@ -27,7 +27,8 @@ abstract class MultiPager<T> : IPager<T> {
 
     val totalPagers: Int get() = _pagers.size;
 
-    constructor(pagers : List<IPager<T>>, allowFailure: Boolean = false) {
+    constructor(pagers : List<IPager<T>>, allowFailure: Boolean = false, pageSize: Int = 9) {
+        this._pageSize = pageSize;
         this.allowFailure = allowFailure;
         _pagers = pagers.toMutableList();
         _subSinglePagers = _pagers.map { SingleItemPager(it) }.toMutableList();

app/src/main/java/com/futo/platformplayer/developer/DeveloperEndpoints.kt

@@ -416,7 +416,7 @@ class DeveloperEndpoints(private val context: Context) {
             val resp = _client.get(body.url!!, body.headers);
 
             context.respondCode(200,
-                Json.encodeToString(PackageHttp.BridgeHttpResponse(resp.code, resp.body?.string())),
+                Json.encodeToString(PackageHttp.BridgeHttpResponse(resp.url, resp.code, resp.body?.string())),
                 context.query.getOrDefault("CT", "text/plain"));
         }
         catch(ex: Exception) {

app/src/main/java/com/futo/platformplayer/dialogs/CommentDialog.kt

@@ -2,12 +2,16 @@ package com.futo.platformplayer.dialogs
 
 import android.app.AlertDialog
 import android.content.Context
+import android.graphics.Color
 import android.os.Bundle
+import android.text.Editable
+import android.text.TextWatcher
 import android.view.LayoutInflater
 import android.view.WindowManager
 import android.view.inputmethod.InputMethodManager
 import android.widget.*
 import com.futo.platformplayer.R
+import com.futo.platformplayer.UIDialogs
 import com.futo.platformplayer.api.media.PlatformID
 import com.futo.platformplayer.api.media.models.PlatformAuthorLink
 import com.futo.platformplayer.api.media.models.comments.IPlatformComment
@@ -32,6 +36,8 @@ class CommentDialog(context: Context?, val contextUrl: String, val ref: Protocol
     private lateinit var _buttonCancel: MaterialButton;
     private lateinit var _editComment: EditText;
     private lateinit var _inputMethodManager: InputMethodManager;
+    private lateinit var _textCharacterCount: TextView;
+    private lateinit var _textCharacterCountMax: TextView;
 
     val onCommentAdded = Event1<IPlatformComment>();
 
@@ -42,6 +48,26 @@ class CommentDialog(context: Context?, val contextUrl: String, val ref: Protocol
         _buttonCancel = findViewById(R.id.button_cancel);
         _buttonCreate = findViewById(R.id.button_create);
         _editComment = findViewById(R.id.edit_comment);
+        _textCharacterCount = findViewById(R.id.character_count);
+        _textCharacterCountMax = findViewById(R.id.character_count_max);
+
+        _editComment.addTextChangedListener(object : TextWatcher {
+            override fun afterTextChanged(s: Editable?) = Unit
+            override fun beforeTextChanged(s: CharSequence?, start: Int, count: Int, after: Int) = Unit
+            override fun onTextChanged(s: CharSequence?, start: Int, before: Int, count: Int) {
+                _textCharacterCount.text = count.toString();
+
+                if (count > PolycentricPlatformComment.MAX_COMMENT_SIZE) {
+                    _textCharacterCount.setTextColor(Color.RED);
+                    _textCharacterCountMax.setTextColor(Color.RED);
+                    _buttonCreate.alpha = 0.4f;
+                } else {
+                    _textCharacterCount.setTextColor(Color.WHITE);
+                    _textCharacterCountMax.setTextColor(Color.WHITE);
+                    _buttonCreate.alpha = 1.0f;
+                }
+            }
+        });
 
         _inputMethodManager = context.getSystemService(Context.INPUT_METHOD_SERVICE) as InputMethodManager;
 
@@ -53,6 +79,11 @@ class CommentDialog(context: Context?, val contextUrl: String, val ref: Protocol
         _buttonCreate.setOnClickListener {
             clearFocus();
 
+            if (_editComment.text.count() > PolycentricPlatformComment.MAX_COMMENT_SIZE) {
+                UIDialogs.toast(context, "Comment should be less than 5000 characters");
+                return@setOnClickListener;
+            }
+
             val comment = _editComment.text.toString();
             val processHandle = StatePolycentric.instance.processHandle!!
             val eventPointer = processHandle.post(comment, null, ref)

app/src/main/java/com/futo/platformplayer/engine/V8Plugin.kt

@@ -51,6 +51,8 @@ class V8Plugin {
      */
     val afterBusy = Event1<Int>();
 
+    val onScriptException = Event1<ScriptException>();
+
     constructor(context: Context, config: IV8PluginConfig, script: String? = null, client: ManagedHttpClient = ManagedHttpClient(), clientAuth: ManagedHttpClient = ManagedHttpClient()) {
         this._client = client;
         this._clientAuth = clientAuth;
@@ -217,7 +219,13 @@ class V8Plugin {
     }
 
     fun <T : Any> catchScriptErrors(context: String, code: String? = null, handle: ()->T): T {
-        return catchScriptErrors(this.config, context, code, handle);
+        try {
+            return catchScriptErrors(this.config, context, code, handle);
+        }
+        catch(ex: ScriptException) {
+            onScriptException.emit(ex);
+            throw ex;
+        }
     }
 
     companion object {
@@ -242,7 +250,7 @@ class V8Plugin {
                 if(result is V8ValueObject) {
                     val type = result.getString("plugin_type");
                     if(type != null && type.endsWith("Exception"))
-                        Companion.throwExceptionFromV8(
+                        throwExceptionFromV8(
                             config,
                             result.getOrThrow(config, "plugin_type", "V8Plugin"),
                             result.getOrThrow(config, "message", "V8Plugin"),
@@ -259,19 +267,28 @@ class V8Plugin {
                 throw ScriptCompilationException(config, "Compilation: ${scriptEx.message}\n(${scriptEx.scriptingError.lineNumber})[${scriptEx.scriptingError.startColumn}-${scriptEx.scriptingError.endColumn}]: ${scriptEx.scriptingError.sourceLine}", null, codeStripped);
             }
             catch(executeEx: JavetExecutionException) {
-                val exMessage = extractJSExceptionMessage(executeEx);
+                if(executeEx.scriptingError?.context?.containsKey("plugin_type") == true) {
+                    val pluginType = executeEx.scriptingError.context["plugin_type"].toString();
 
-                if(executeEx.scriptingError?.context?.containsKey("plugin_type") == true)
+                    //Captcha
+                    if (pluginType == "CaptchaRequiredException") {
+                        throw ScriptCaptchaRequiredException(config,
+                            executeEx.scriptingError.context["url"]?.toString(),
+                            executeEx.scriptingError.context["body"]?.toString(),
+                            executeEx, executeEx.scriptingError?.stack, codeStripped);
+                    }
+
+                    //Others
                     throwExceptionFromV8(
                         config,
-                        executeEx.scriptingError.context["plugin_type"].toString(),
-                        (exMessage ?: ""),
+                        pluginType,
+                        (extractJSExceptionMessage(executeEx) ?: ""),
                         executeEx,
                         executeEx.scriptingError?.stack,
                         codeStripped
                     );
-
-                throw ScriptExecutionException(config, "${exMessage}", null, executeEx.scriptingError?.stack, codeStripped);
+                }
+                throw ScriptExecutionException(config, extractJSExceptionMessage(executeEx) ?: "", null, executeEx.scriptingError?.stack, codeStripped);
             }
             catch(ex: Exception) {
                 throw ex;

app/src/main/java/com/futo/platformplayer/engine/exceptions/ScriptCaptchaRequiredException.kt

@@ -0,0 +1,18 @@
+package com.futo.platformplayer.engine.exceptions
+
+import com.caoccao.javet.values.reference.V8ValueObject
+import com.futo.platformplayer.engine.IV8PluginConfig
+import com.futo.platformplayer.getOrDefault
+import com.futo.platformplayer.getOrThrow
+
+class ScriptCaptchaRequiredException(config: IV8PluginConfig, val url: String?, val body: String?, ex: Exception? = null, stack: String? = null, code: String? = null) : ScriptException(config, "Captcha required", ex, stack, code) {
+
+    companion object {
+        fun fromV8(config: IV8PluginConfig, obj: V8ValueObject) : ScriptException {
+            val contextName = "ScriptCaptchaRequiredException";
+            return ScriptCaptchaRequiredException(config,
+                obj.getOrDefault<String>(config, "url", contextName, null),
+                obj.getOrDefault<String>(config, "body", contextName, null));
+        }
+    }
+}

app/src/main/java/com/futo/platformplayer/engine/packages/PackageHttp.kt

@@ -108,11 +108,12 @@ class PackageHttp: V8Package {
     }
 
     @kotlinx.serialization.Serializable
-    class BridgeHttpResponse(val code: Int, val body: String?, val headers: Map<String, List<String>>? = null) : IV8Convertable {
+    class BridgeHttpResponse(val url: String, val code: Int, val body: String?, val headers: Map<String, List<String>>? = null) : IV8Convertable {
         val isOk = code >= 200 && code < 300;
 
         override fun toV8(runtime: V8Runtime): V8Value? {
             val obj = runtime.createV8ValueObject();
+            obj.set("url", url);
             obj.set("code", code);
             obj.set("body", body);
             obj.set("headers", headers);
@@ -227,7 +228,7 @@ class PackageHttp: V8Package {
                     val resp = client.requestMethod(method, url, headers);
                     val responseBody = resp.body?.string();
                     logResponse(method, url, resp.code, resp.headers, responseBody);
-                    return@catchHttp BridgeHttpResponse(resp.code, responseBody, sanitizeResponseHeaders(resp.headers));
+                    return@catchHttp BridgeHttpResponse(resp.url, resp.code, responseBody, sanitizeResponseHeaders(resp.headers));
                 }
             };
         }
@@ -241,7 +242,7 @@ class PackageHttp: V8Package {
                     val resp = client.requestMethod(method, url, body, headers);
                     val responseBody = resp.body?.string();
                     logResponse(method, url, resp.code, resp.headers, responseBody);
-                    return@catchHttp BridgeHttpResponse(resp.code, responseBody, sanitizeResponseHeaders(resp.headers));
+                    return@catchHttp BridgeHttpResponse(resp.url, resp.code, responseBody, sanitizeResponseHeaders(resp.headers));
                 }
             };
         }
@@ -256,7 +257,7 @@ class PackageHttp: V8Package {
                     val resp = client.get(url, headers);
                     val responseBody = resp.body?.string();
                     logResponse("GET", url, resp.code, resp.headers, responseBody);
-                    return@catchHttp BridgeHttpResponse(resp.code, responseBody, sanitizeResponseHeaders(resp.headers));
+                    return@catchHttp BridgeHttpResponse(resp.url, resp.code, responseBody, sanitizeResponseHeaders(resp.headers));
                 }
             };
         }
@@ -270,7 +271,7 @@ class PackageHttp: V8Package {
                     val resp = client.post(url, body, headers);
                     val responseBody = resp.body?.string();
                     logResponse("POST", url, resp.code, resp.headers, responseBody);
-                    return@catchHttp BridgeHttpResponse(resp.code, responseBody, sanitizeResponseHeaders(resp.headers));
+                    return@catchHttp BridgeHttpResponse(resp.url, resp.code, responseBody, sanitizeResponseHeaders(resp.headers));
                 }
             };
         }
@@ -367,7 +368,7 @@ class PackageHttp: V8Package {
             }
             //Forward timeouts
             catch(ex: SocketTimeoutException) {
-                return BridgeHttpResponse(408, null);
+                return BridgeHttpResponse("", 408, null);
             }
         }
     }
@@ -461,7 +462,7 @@ class PackageHttp: V8Package {
         }
         //Forward timeouts
         catch(ex: SocketTimeoutException) {
-            return BridgeHttpResponse(408, null);
+            return BridgeHttpResponse("", 408, null);
         }
     }
 

app/src/main/java/com/futo/platformplayer/exceptions/RateLimitException.kt

@@ -0,0 +1,9 @@
+package com.futo.platformplayer.exceptions
+
+class RateLimitException : Throwable {
+    val pluginIds: List<String>;
+
+    constructor(pluginIds: List<String>): super() {
+        this.pluginIds = pluginIds ?: listOf();
+    }
+}

app/src/main/java/com/futo/platformplayer/fragment/channel/tab/ChannelAboutFragment.kt

@@ -77,7 +77,7 @@ class ChannelAboutFragment : Fragment, IChannelTabFragment {
         };
         _textName?.text = channel.name;
 
-        val metadata = "${channel.subscribers.toHumanNumber()} subscribers";
+        val metadata = if(channel.subscribers > 0) "${channel.subscribers.toHumanNumber()} subscribers" else "";
         _textMetadata?.text = metadata;
         _lastChannel = channel;
         setLinks(channel.links, channel.name);

app/src/main/java/com/futo/platformplayer/fragment/channel/tab/ChannelContentsFragment.kt

@@ -29,6 +29,7 @@ import com.futo.platformplayer.constructs.Event1
 import com.futo.platformplayer.constructs.Event2
 import com.futo.platformplayer.constructs.TaskHandler
 import com.futo.platformplayer.engine.exceptions.PluginException
+import com.futo.platformplayer.engine.exceptions.ScriptCaptchaRequiredException
 import com.futo.platformplayer.fragment.mainactivity.main.FeedView
 import com.futo.platformplayer.fragment.mainactivity.main.PolycentricProfile
 import com.futo.platformplayer.states.StatePolycentric
@@ -76,7 +77,9 @@ class ChannelContentsFragment : Fragment(), IChannelTabFragment {
         }).success {
             setLoading(false);
             setPager(it);
-        }.exception<Throwable> {
+        }
+        .exception<ScriptCaptchaRequiredException> {  }
+        .exception<Throwable> {
             Logger.w(TAG, "Failed to load initial videos.", it);
             UIDialogs.showGeneralRetryErrorDialog(requireContext(), it.message ?: "", it, { loadNextPage() });
     };

app/src/main/java/com/futo/platformplayer/fragment/channel/tab/ChannelListFragment.kt

@@ -14,6 +14,7 @@ import com.futo.platformplayer.api.media.models.PlatformAuthorLink
 import com.futo.platformplayer.api.media.models.channels.IPlatformChannel
 import com.futo.platformplayer.constructs.Event1
 import com.futo.platformplayer.constructs.TaskHandler
+import com.futo.platformplayer.engine.exceptions.ScriptCaptchaRequiredException
 import com.futo.platformplayer.fragment.mainactivity.main.ChannelFragment
 import com.futo.platformplayer.fragment.mainactivity.main.PolycentricProfile
 import com.futo.platformplayer.logging.Logger
@@ -52,7 +53,8 @@ class ChannelListFragment : Fragment, IChannelTabFragment {
         _authorLinks.add(PlatformAuthorLink(it.id, it.name, it.url, it.thumbnail));
         adapter.notifyItemInserted(adapter.childToParentPosition(_authorLinks.size - 1));
         loadNext();
-    }.exceptionWithParameter<Throwable> { ex, para ->
+    }.exception<ScriptCaptchaRequiredException> {  }
+        .exceptionWithParameter<Throwable> { ex, para ->
         Logger.w(ChannelFragment.TAG, "Failed to load results.", ex);
         UIDialogs.toast(requireContext(), "Failed to fetch\n${para}", false)
         loadNext();

app/src/main/java/com/futo/platformplayer/fragment/mainactivity/main/ChannelFragment.kt

@@ -361,7 +361,7 @@ class ChannelFragment : MainFragment() {
 
             _buttonSubscribe.setSubscribeChannel(channel);
             _textChannel.text = channel.name;
-            _textChannelSub.text = "${channel.subscribers.toHumanNumber()} subscribers";
+            _textChannelSub.text = if(channel.subscribers > 0) "${channel.subscribers.toHumanNumber()} subscribers" else "";
 
             _creatorThumbnail.setThumbnail(channel.thumbnail, true);
             Glide.with(_imageBanner)

app/src/main/java/com/futo/platformplayer/fragment/mainactivity/main/ContentSearchResultsFragment.kt

@@ -15,6 +15,7 @@ import com.futo.platformplayer.api.media.models.ResultCapabilities
 import com.futo.platformplayer.api.media.models.contents.IPlatformContent
 import com.futo.platformplayer.api.media.structures.IPager
 import com.futo.platformplayer.constructs.TaskHandler
+import com.futo.platformplayer.engine.exceptions.ScriptCaptchaRequiredException
 import com.futo.platformplayer.fragment.mainactivity.topbar.SearchTopBarFragment
 import com.futo.platformplayer.views.FeedStyle
 import kotlinx.coroutines.Dispatchers
@@ -86,7 +87,7 @@ class ContentSearchResultsFragment : MainFragment() {
                     StatePlatform.instance.searchRefresh(fragment.lifecycleScope, query, null, _sortBy, _filterValues, _enabledClientIds)
                 }
             })
-            .success { loadedResult(it); }
+            .success { loadedResult(it); }.exception<ScriptCaptchaRequiredException> {  }
             .exception<Throwable> {
                 Logger.w(ChannelFragment.TAG, "Failed to load results.", it);
                 UIDialogs.showGeneralRetryErrorDialog(context, it.message ?: "", it, { loadResults() });

app/src/main/java/com/futo/platformplayer/fragment/mainactivity/main/CreatorSearchResultsFragment.kt

@@ -13,6 +13,7 @@ import com.futo.platformplayer.Settings
 import com.futo.platformplayer.api.media.models.PlatformAuthorLink
 import com.futo.platformplayer.api.media.structures.IPager
 import com.futo.platformplayer.constructs.TaskHandler
+import com.futo.platformplayer.engine.exceptions.ScriptCaptchaRequiredException
 import com.futo.platformplayer.fragment.mainactivity.topbar.SearchTopBarFragment
 import com.futo.platformplayer.views.FeedStyle
 
@@ -56,6 +57,7 @@ class CreatorSearchResultsFragment : MainFragment() {
         constructor(fragment: CreatorSearchResultsFragment, inflater: LayoutInflater): super(fragment, inflater) {
             _taskSearch = TaskHandler<String, IPager<PlatformAuthorLink>>({fragment.lifecycleScope}, { query -> StatePlatform.instance.searchChannels(query) })
                 .success { loadedResult(it); }
+                .exception<ScriptCaptchaRequiredException> {  }
                 .exception<Throwable> {
                     Logger.w(ChannelFragment.TAG, "Failed to load results.", it);
                     UIDialogs.showGeneralRetryErrorDialog(context, it.message ?: "", it, { loadResults() });

app/src/main/java/com/futo/platformplayer/fragment/mainactivity/main/HomeFragment.kt

@@ -8,21 +8,27 @@ import android.view.ViewGroup
 import androidx.lifecycle.lifecycleScope
 import androidx.recyclerview.widget.LinearLayoutManager
 import com.futo.platformplayer.*
+import com.futo.platformplayer.activities.CaptchaActivity
 import com.futo.platformplayer.api.media.IPlatformClient
 import com.futo.platformplayer.api.media.models.contents.IPlatformContent
 import com.futo.platformplayer.api.media.models.video.IPlatformVideo
 import com.futo.platformplayer.api.media.platforms.js.JSClient
+import com.futo.platformplayer.api.media.platforms.js.internal.JSHttpClient
 import com.futo.platformplayer.api.media.structures.EmptyPager
 import com.futo.platformplayer.api.media.structures.IPager
 import com.futo.platformplayer.constructs.TaskHandler
+import com.futo.platformplayer.engine.exceptions.ScriptCaptchaRequiredException
 import com.futo.platformplayer.engine.exceptions.ScriptExecutionException
 import com.futo.platformplayer.engine.exceptions.ScriptImplementationException
 import com.futo.platformplayer.fragment.mainactivity.topbar.ImportTopBarFragment
 import com.futo.platformplayer.logging.Logger
+import com.futo.platformplayer.others.CaptchaWebViewClient
 import com.futo.platformplayer.states.AnnouncementType
 import com.futo.platformplayer.states.StateAnnouncement
 import com.futo.platformplayer.states.StateMeta
 import com.futo.platformplayer.states.StatePlatform
+import com.futo.platformplayer.states.StatePlugins
+import com.futo.platformplayer.states.StateSubscriptions
 import com.futo.platformplayer.views.announcements.AnnouncementView
 import com.futo.platformplayer.views.FeedStyle
 import com.futo.platformplayer.views.adapters.ContentPreviewViewHolder
@@ -92,6 +98,7 @@ class HomeFragment : MainFragment() {
                 StatePlatform.instance.getHomeRefresh(fragment.lifecycleScope)
             })
             .success { loadedResult(it); }
+            .exception<ScriptCaptchaRequiredException> {  }
             .exception<ScriptExecutionException> {
                 Logger.w(ChannelFragment.TAG, "Plugin failure.", it);
                 UIDialogs.showDialog(context, R.drawable.ic_error_pred, "Failed to get Home\nPlugin [${it.config.name}]", it.message, null, 0,
@@ -100,14 +107,14 @@ class HomeFragment : MainFragment() {
                 );
             }
             .exception<ScriptImplementationException> {
-                Logger.w(ChannelFragment.TAG, "Plugin failure.", it);
+                Logger.w(TAG, "Plugin failure.", it);
                 UIDialogs.showDialog(context, R.drawable.ic_error_pred, "Failed to get Home\nPlugin [${it.config.name}]", it.message, null, 0,
                     UIDialogs.Action("Ignore", {}),
                     UIDialogs.Action("Sources", { fragment.navigate<SourcesFragment>() }, UIDialogs.ActionStyle.PRIMARY)
                 );
             }
             .exception<Throwable> {
-                Logger.w(ChannelFragment.TAG, "Failed to load channel.", it);
+                Logger.w(TAG, "Failed to load channel.", it);
                 UIDialogs.showGeneralRetryErrorDialog(context, "Failed to get Home", it, {
                     loadResults()
                 }) {
@@ -134,6 +141,8 @@ class HomeFragment : MainFragment() {
             } else {
                 setLoading(false);
             }
+
+            finishRefreshLayoutLoader();
         }
 
         override fun reload() {

app/src/main/java/com/futo/platformplayer/fragment/mainactivity/main/PostDetailFragment.kt

@@ -346,24 +346,24 @@ class PostDetailFragment : MainFragment {
 
                         _rating.visibility = VISIBLE;
                         _rating.setRating(RatingLikeDislikes(likes, dislikes), hasLiked, hasDisliked);
-                        _rating.onLikeDislikeUpdated.subscribe(this) { processHandle, newHasLiked, newHasDisliked ->
-                            if (newHasLiked) {
-                                processHandle.opinion(ref, Opinion.like);
-                            } else if (newHasDisliked) {
-                                processHandle.opinion(ref, Opinion.dislike);
+                        _rating.onLikeDislikeUpdated.subscribe(this) { args ->
+                            if (args.hasLiked) {
+                                args.processHandle.opinion(ref, Opinion.like);
+                            } else if (args.hasDisliked) {
+                                args.processHandle.opinion(ref, Opinion.dislike);
                             } else {
-                                processHandle.opinion(ref, Opinion.neutral);
+                                args.processHandle.opinion(ref, Opinion.neutral);
                             }
 
                             StateApp.instance.scopeOrNull?.launch(Dispatchers.IO) {
                                 try {
-                                    processHandle.fullyBackfillServers();
+                                    args.processHandle.fullyBackfillServers();
                                 } catch (e: Throwable) {
                                     Logger.e(TAG, "Failed to backfill servers", e)
                                 }
                             }
 
-                            StatePolycentric.instance.updateLikeMap(ref, newHasLiked, newHasDisliked)
+                            StatePolycentric.instance.updateLikeMap(ref, args.hasLiked, args.hasDisliked)
                         };
                     }
                 } catch (e: Throwable) {

app/src/main/java/com/futo/platformplayer/fragment/mainactivity/main/SubscriptionsFeedFragment.kt

@@ -12,13 +12,16 @@ import com.futo.platformplayer.*
 import com.futo.platformplayer.api.media.models.contents.ContentType
 import com.futo.platformplayer.api.media.models.contents.IPlatformContent
 import com.futo.platformplayer.api.media.models.video.IPlatformVideo
+import com.futo.platformplayer.api.media.platforms.js.JSClient
 import com.futo.platformplayer.api.media.structures.IPager
 import com.futo.platformplayer.cache.ChannelContentCache
 import com.futo.platformplayer.constructs.TaskHandler
 import com.futo.platformplayer.engine.exceptions.PluginException
 import com.futo.platformplayer.exceptions.ChannelException
+import com.futo.platformplayer.exceptions.RateLimitException
 import com.futo.platformplayer.logging.Logger
 import com.futo.platformplayer.states.StateApp
+import com.futo.platformplayer.states.StatePlatform
 import com.futo.platformplayer.states.StateSubscriptions
 import com.futo.platformplayer.stores.FragmentedStorage
 import com.futo.platformplayer.stores.FragmentedStorageFileJson
@@ -31,6 +34,7 @@ import com.futo.platformplayer.views.subscriptions.SubscriptionBar
 import kotlinx.coroutines.CancellationException
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.launch
+import kotlinx.coroutines.withContext
 import kotlinx.serialization.Serializable
 import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
@@ -130,6 +134,10 @@ class SubscriptionsFeedFragment : MainFragment() {
                     };
                 }
             }
+
+            if (!StateSubscriptions.instance.isGlobalUpdating) {
+                finishRefreshLayoutLoader();
+            }
         }
 
         override fun cleanup() {
@@ -156,8 +164,17 @@ class SubscriptionsFeedFragment : MainFragment() {
         private val _filterLock = Object();
         private val _filterSettings = FragmentedStorage.get<FeedFilterSettings>("subFeedFilter");
 
+        private var _bypassRateLimit = false;
         private val _lastExceptions: List<Throwable>? = null;
         private val _taskGetPager = TaskHandler<Boolean, IPager<IPlatformContent>>({StateApp.instance.scope}, { withRefresh ->
+            if(!_bypassRateLimit) {
+                val subRequestCounts = StateSubscriptions.instance.getSubscriptionRequestCount();
+                val reqCountStr = subRequestCounts.map { "    ${it.key.config.name}: ${it.value}/${it.key.config.subscriptionRateLimit}" }.joinToString("\n");
+                val rateLimitPlugins = subRequestCounts.filter { clientCount -> clientCount.key.config.subscriptionRateLimit?.let { rateLimit -> clientCount.value > rateLimit } == true }
+                Logger.w(TAG, "Refreshing subscriptions with requests:\n" + reqCountStr);
+                if(rateLimitPlugins.any())
+                    throw RateLimitException(rateLimitPlugins.map { it.key.id });
+            }
             val resp = StateSubscriptions.instance.getGlobalSubscriptionFeed(StateApp.instance.scope, withRefresh);
 
             val currentExs = StateSubscriptions.instance.globalSubscriptionExceptions;
@@ -167,6 +184,29 @@ class SubscriptionsFeedFragment : MainFragment() {
             return@TaskHandler resp;
         })
             .success { loadedResult(it); }
+            .exception<RateLimitException> {
+                fragment.lifecycleScope.launch(Dispatchers.IO) {
+                    val subs = StateSubscriptions.instance.getSubscriptions();
+                    val subsByLimited = it.pluginIds.map{ StatePlatform.instance.getClientOrNull(it) }
+                        .filterIsInstance<JSClient>()
+                        .associateWith { client -> subs.filter { it.getClient() == client } }
+                        .map { Pair(it.key, it.value) }
+
+                    withContext(Dispatchers.Main) {
+                        UIDialogs.showDialog(context, R.drawable.ic_security_pred,
+                            "Rate Limit Warning",  "This is a temporary measure to prevent people from hitting rate limit until we have better support for lots of subscriptions." +
+                                    "\n\nYou have too many subscriptions for the following plugins:\n",
+                            subsByLimited.map { "${it.first.config.name}: ${it.second.size} Subscriptions" } .joinToString("\n"), 0, UIDialogs.Action("Refresh Anyway", {
+                                _bypassRateLimit = true;
+                                loadResults();
+                            }, UIDialogs.ActionStyle.DANGEROUS_TEXT),
+                            UIDialogs.Action("OK", {
+                                finishRefreshLayoutLoader();
+                                setLoading(false);
+                            }, UIDialogs.ActionStyle.PRIMARY));
+                    }
+                }
+            }
             .exception<Throwable> {
                 Logger.w(ChannelFragment.TAG, "Failed to load channel.", it);
                 if(it !is CancellationException)

app/src/main/java/com/futo/platformplayer/fragment/mainactivity/main/VideoDetailView.kt

@@ -1042,24 +1042,24 @@ class VideoDetailView : ConstraintLayout {
                     withContext(Dispatchers.Main) {
                         _rating.visibility = View.VISIBLE;
                         _rating.setRating(RatingLikeDislikes(likes, dislikes), hasLiked, hasDisliked);
-                        _rating.onLikeDislikeUpdated.subscribe(this) { processHandle, newHasLiked, newHasDisliked ->
-                            if (newHasLiked) {
-                                processHandle.opinion(ref, Opinion.like);
-                            } else if (newHasDisliked) {
-                                processHandle.opinion(ref, Opinion.dislike);
+                        _rating.onLikeDislikeUpdated.subscribe(this) { args ->
+                            if (args.hasLiked) {
+                                args.processHandle.opinion(ref, Opinion.like);
+                            } else if (args.hasDisliked) {
+                                args.processHandle.opinion(ref, Opinion.dislike);
                             } else {
-                                processHandle.opinion(ref, Opinion.neutral);
+                                args.processHandle.opinion(ref, Opinion.neutral);
                             }
 
                             fragment.lifecycleScope.launch(Dispatchers.IO) {
                                 try {
-                                    processHandle.fullyBackfillServers();
+                                    args.processHandle.fullyBackfillServers();
                                 } catch (e: Throwable) {
                                     Logger.e(TAG, "Failed to backfill servers", e)
                                 }
                             }
 
-                            StatePolycentric.instance.updateLikeMap(ref, newHasLiked, newHasDisliked)
+                            StatePolycentric.instance.updateLikeMap(ref, args.hasLiked, args.hasDisliked)
                         };
                     }
                 } catch (e: Throwable) {
@@ -1610,10 +1610,10 @@ class VideoDetailView : ConstraintLayout {
         _lastSubtitleSource = toSet;
     }
 
-    private fun handleUnavailableVideo() {
+    private fun handleUnavailableVideo(msg: String? = null) {
         if (!nextVideo()) {
             if(video?.datetime == null || video?.datetime!! < OffsetDateTime.now().minusHours(1))
-                UIDialogs.showDialog(context, R.drawable.ic_lock, "Unavailable video", "This video is unavailable.", null, 0,
+                UIDialogs.showDialog(context, R.drawable.ic_lock, "Unavailable video", msg ?: "This video is unavailable.", null, 0,
                     UIDialogs.Action("Back", {
                         this@VideoDetailView.onClose.emit();
                     }, UIDialogs.ActionStyle.PRIMARY));
@@ -2092,7 +2092,7 @@ class VideoDetailView : ConstraintLayout {
         }
         .exception<ScriptUnavailableException> {
             Logger.w(TAG, "exception<ScriptUnavailableException>", it);
-            handleUnavailableVideo();
+            handleUnavailableVideo(it.message);
         }
         .exception<ScriptException> {
             Logger.w(TAG, "exception<ScriptException>", it)

app/src/main/java/com/futo/platformplayer/models/Subscription.kt

@@ -3,24 +3,48 @@ package com.futo.platformplayer.models
 import com.futo.platformplayer.api.media.models.channels.IPlatformChannel
 import com.futo.platformplayer.api.media.models.channels.SerializedChannel
 import com.futo.platformplayer.api.media.models.contents.IPlatformContent
+import com.futo.platformplayer.getNowDiffDays
 import com.futo.platformplayer.serializers.OffsetDateTimeSerializer
+import com.futo.platformplayer.states.StatePlatform
 import java.time.OffsetDateTime
 
 @kotlinx.serialization.Serializable
 class Subscription {
     var channel: SerializedChannel;
 
+    //Last found content
     @kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class)
     var lastVideo : OffsetDateTime = OffsetDateTime.MAX;
     @kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class)
     var lastLiveStream : OffsetDateTime = OffsetDateTime.MAX;
+    @kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class)
+    var lastPost : OffsetDateTime = OffsetDateTime.MAX;
 
+    //Last update time
+    @kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class)
+    var lastVideoUpdate : OffsetDateTime = OffsetDateTime.MIN;
+    @kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class)
+    var lastStreamUpdate : OffsetDateTime = OffsetDateTime.MIN;
+    @kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class)
+    var lastLiveStreamUpdate : OffsetDateTime = OffsetDateTime.MIN;
+    @kotlinx.serialization.Serializable(with = OffsetDateTimeSerializer::class)
+    var lastPostUpdate : OffsetDateTime = OffsetDateTime.MIN;
+
+    //Last video interval
     var uploadInterval : Int = 0;
+    var uploadPostInterval : Int = 0;
+
 
     constructor(channel : SerializedChannel) {
         this.channel = channel;
     }
 
+    fun shouldFetchStreams() = lastLiveStream.getNowDiffDays() < 7;
+    fun shouldFetchLiveStreams() = lastLiveStream.getNowDiffDays() < 14;
+    fun shouldFetchPosts() = lastPost.getNowDiffDays() < 2;
+
+    fun getClient() = StatePlatform.instance.getChannelClientOrNull(channel.url);
+
     fun updateChannel(channel: IPlatformChannel) {
         this.channel = SerializedChannel.fromChannel(channel);
     }

app/src/main/java/com/futo/platformplayer/models/Telemetry.kt

@@ -9,7 +9,6 @@ data class Telemetry(
     val buildType: String,
     val debug: Boolean,
     val isUnstableBuild: Boolean,
-    val time: Long,
     val brand: String,
     val manufacturer: String,
     val model: String

app/src/main/java/com/futo/platformplayer/others/CaptchaWebViewClient.kt

@@ -0,0 +1,76 @@
+package com.futo.platformplayer.others
+
+import android.webkit.*
+import com.futo.platformplayer.api.media.Serializer
+import com.futo.platformplayer.api.media.platforms.js.SourceCaptchaData
+import com.futo.platformplayer.api.media.platforms.js.SourcePluginAuthConfig
+import com.futo.platformplayer.api.media.platforms.js.SourcePluginCaptchaConfig
+import com.futo.platformplayer.api.media.platforms.js.SourcePluginConfig
+import com.futo.platformplayer.constructs.Event1
+import com.futo.platformplayer.constructs.Event2
+import com.futo.platformplayer.logging.Logger
+import kotlinx.serialization.encodeToString
+
+class CaptchaWebViewClient : WebViewClient {
+    val onCaptchaFinished = Event1<SourceCaptchaData?>();
+    val onPageLoaded = Event2<WebView?, String?>()
+
+    private val _pluginConfig: SourcePluginConfig?;
+    private val _captchaConfig: SourcePluginCaptchaConfig;
+
+    private var _didNotify = false;
+    private val _extractor: WebViewRequirementExtractor;
+
+    constructor(config: SourcePluginConfig) : super() {
+        _pluginConfig = config;
+        _captchaConfig = config.captcha!!;
+        _extractor = WebViewRequirementExtractor(
+            config.allowUrls,
+            null,
+            null,
+            config.captcha!!.cookiesToFind,
+            config.captcha!!.completionUrl,
+            config.captcha!!.cookiesExclOthers
+        );
+        Logger.i(TAG, "Captcha [${config.name}]" +
+                "\nRequired Cookies: ${Serializer.json.encodeToString(config.captcha!!.cookiesToFind)}",);
+    }
+    constructor(captcha: SourcePluginCaptchaConfig) : super() {
+        _pluginConfig = null;
+        _captchaConfig = captcha;
+        _extractor = WebViewRequirementExtractor(
+            null,
+            null,
+            null,
+            captcha.cookiesToFind,
+            captcha.completionUrl,
+            captcha.cookiesExclOthers
+        );
+    }
+
+    override fun onPageFinished(view: WebView?, url: String?) {
+        super.onPageFinished(view, url);
+        Logger.i(TAG, "onPageFinished url = ${url}")
+        onPageLoaded.emit(view, url);
+    }
+
+    override fun shouldInterceptRequest(view: WebView?, request: WebResourceRequest?): WebResourceResponse? {
+        if(request == null)
+            return super.shouldInterceptRequest(view, request as WebResourceRequest?);
+
+        val extracted = _extractor.handleRequest(view, request);
+        if(extracted != null && !_didNotify) {
+            _didNotify = true;
+            onCaptchaFinished.emit(SourceCaptchaData(
+               extracted.cookies,
+               extracted.headers
+            ));
+        }
+
+        return super.shouldInterceptRequest(view, request);
+    }
+
+    companion object {
+        private val TAG = "CaptchaWebViewClient";
+    }
+}

app/src/main/java/com/futo/platformplayer/others/LoginWebViewClient.kt

@@ -12,6 +12,7 @@ import com.futo.platformplayer.constructs.Event2
 import com.futo.platformplayer.logging.Logger
 import com.futo.platformplayer.matchesDomain
 import kotlinx.serialization.encodeToString
+import kotlinx.serialization.json.Json
 
 class LoginWebViewClient : WebViewClient {
     private val LOG_VERBOSE = false;
@@ -46,6 +47,7 @@ class LoginWebViewClient : WebViewClient {
         onPageLoaded.emit(view, url);
     }
 
+    //TODO: Use new WebViewRequirementExtractor when time to test extensively
     override fun shouldInterceptRequest(view: WebView?, request: WebResourceRequest?): WebResourceResponse? {
         if(request == null)
             return super.shouldInterceptRequest(view, request as WebResourceRequest?);

app/src/main/java/com/futo/platformplayer/others/WebViewRequirementExtractor.kt

@@ -0,0 +1,125 @@
+package com.futo.platformplayer.others
+
+import android.net.Uri
+import android.webkit.CookieManager
+import android.webkit.WebResourceRequest
+import android.webkit.WebView
+import com.futo.platformplayer.api.media.platforms.js.SourceAuth
+import com.futo.platformplayer.logging.Logger
+import com.futo.platformplayer.matchesDomain
+
+class WebViewRequirementExtractor {
+    private val allowedUrls: List<String>;
+    private val headersToFind: List<String>?;
+    private val domainHeadersToFind: Map<String, List<String>>?;
+    private val cookiesToFind: List<String>?;
+    private val completionUrl: String?;
+
+    private val exclOtherCookies: Boolean;
+
+
+    private val headersFoundMap: HashMap<String, HashMap<String, String>> = hashMapOf();
+    private val cookiesFoundMap = hashMapOf<String, HashMap<String, String>>();
+    private var urlFound = false;
+
+
+    constructor(allowedUrls: List<String>?, headers: List<String>?, domainHeaders: Map<String, List<String>>?, cookies: List<String>?, url: String?, exclOtherCookies: Boolean = false) {
+        this.allowedUrls = allowedUrls ?: listOf("everywhere");
+        this.exclOtherCookies = exclOtherCookies;
+        headersToFind = headers;
+        domainHeadersToFind = domainHeaders;
+        cookiesToFind = cookies;
+        completionUrl = url;
+    }
+
+
+    fun handleRequest(view: WebView?, request: WebResourceRequest, logVerbose: Boolean = false): ExtractedData? {
+
+        val domain = request.url.host;
+        val domainLower = request.url.host?.lowercase();
+        if(completionUrl == null)
+            urlFound = true;
+        else urlFound = urlFound || request.url == Uri.parse(completionUrl);
+
+        //HEADERS
+        if(domainLower != null) {
+            val headersToFind = ((headersToFind?.map { Pair(it.lowercase(), domainLower) } ?: listOf()) +
+                    (domainHeadersToFind?.filter { domainLower.matchesDomain(it.key.lowercase())}
+                        ?.flatMap { it.value.map { header -> Pair(header.lowercase(), it.key.lowercase()) } } ?: listOf()));
+
+            val foundHeaders = request.requestHeaders.filter { requestHeader -> headersToFind.any { it.first.equals(requestHeader.key, true)} &&
+                    (!requestHeader.key.equals("Authorization", ignoreCase = true) || requestHeader.value != "undefined") } //TODO: More universal fix (optional regex?)
+            for(header in foundHeaders) {
+                for(headerDomain in headersToFind.filter { it.first.equals(header.key, true) }) {
+                    if (!headersFoundMap.containsKey(headerDomain.second))
+                        headersFoundMap[headerDomain.second] = hashMapOf();
+                    headersFoundMap[headerDomain.second]!![header.key.lowercase()] = header.value;
+                }
+            }
+        }
+
+
+        //COOKIES
+        //TODO: This is not an ideal solution, we want to intercept the response, but interception need to be rewritten to support that. Correct implementation commented underneath
+        //TODO: For now we assume cookies are legit for all subdomains of a top-level domain, this is the most common scenario anyway
+        val cookieString = CookieManager.getInstance().getCookie(request.url.toString());
+        if(cookieString != null) {
+            val domainParts = domain!!.split(".");
+            val cookieDomain = "." + domainParts.drop(domainParts.size - 2).joinToString(".");
+            if(allowedUrls.any { it == "everywhere" || it.lowercase().matchesDomain(cookieDomain) })
+                cookiesToFind?.let { cookiesToFind ->
+                    val cookies = cookieString.split(";");
+                    for(cookieStr in cookies) {
+                        val cookieSplitIndex = cookieStr.indexOf("=");
+                        if(cookieSplitIndex <= 0) continue;
+                        val cookieKey = cookieStr.substring(0, cookieSplitIndex).trim();
+                        val cookieVal = cookieStr.substring(cookieSplitIndex + 1).trim();
+
+                        if (exclOtherCookies && !cookiesToFind.contains(cookieKey))
+                            continue;
+
+                        if (cookiesFoundMap.containsKey(cookieDomain))
+                            cookiesFoundMap[cookieDomain]!![cookieKey] = cookieVal;
+                        else
+                            cookiesFoundMap[cookieDomain] = hashMapOf(Pair(cookieKey, cookieVal));
+                    }
+                };
+        }
+
+        val headersFound = headersToFind?.map { it.lowercase() }?.all { reqHeader -> headersFoundMap.any { it.value.containsKey(reqHeader) } } ?: true
+        val domainHeadersFound = domainHeadersToFind?.all {
+            if(it.value.isEmpty())
+                return@all true;
+            if(!headersFoundMap.containsKey(it.key.lowercase()))
+                return@all false;
+            val foundDomainHeaders = headersFoundMap[it.key.lowercase()] ?: mapOf();
+            return@all it.value.all { reqHeader -> foundDomainHeaders.containsKey(reqHeader.lowercase()) };
+        } ?: true;
+        val cookiesFound = cookiesToFind?.all { toFind -> cookiesFoundMap.any { it.value.containsKey(toFind) } } ?: true;
+
+        if(logVerbose) {
+            val builder = StringBuilder();
+            builder.appendLine("Request (method: ${request.method}, host: ${request.url.host}, url: ${request.url}, path: ${request.url.path}):");
+            for (pair in request.requestHeaders) {
+                builder.appendLine(" ${pair.key}: ${pair.value}");
+            }
+            builder.appendLine(" Cookies: ${cookiesFoundMap.values.sumOf { it.values.size }}");
+            Logger.i(TAG, builder.toString());
+            Logger.i(TAG, "Result (urlFound: $urlFound, headersFound: $headersFound, cookiesFound: $cookiesFound)");
+        }
+
+        if (urlFound && headersFound && domainHeadersFound && cookiesFound)
+            return ExtractedData(cookiesFoundMap, headersFoundMap);
+        return null;
+    }
+
+
+
+    data class ExtractedData(
+        val cookies: HashMap<String, HashMap<String, String>>,
+        val headers: HashMap<String, HashMap<String, String>>
+    );
+    companion object {
+        val TAG = "WebViewRequirementExtractor";
+    }
+}

app/src/main/java/com/futo/platformplayer/serializers/PlatformContentSerializer.kt

@@ -20,10 +20,10 @@ class PlatformContentSerializer() : JsonContentPolymorphicSerializer<SerializedP
         if(obj?.jsonPrimitive?.isString ?: true)
             return when(obj?.jsonPrimitive?.contentOrNull) {
                 "MEDIA" -> SerializedPlatformVideo.serializer();
-                "NESTED" -> SerializedPlatformNestedContent.serializer();
+                "NESTED_VIDEO" -> SerializedPlatformNestedContent.serializer();
                 "ARTICLE" -> throw NotImplementedError("Articles not yet implemented");
                 "POST" -> throw NotImplementedError("Post not yet implemented");
-                else -> throw NotImplementedError("Unknown Content Type Value: ${obj?.jsonPrimitive?.int}")
+                else -> throw NotImplementedError("Unknown Content Type Value: ${obj?.jsonPrimitive?.contentOrNull}")
             };
         else
             return when(obj?.jsonPrimitive?.int) {

app/src/main/java/com/futo/platformplayer/states/StateApp.kt

@@ -25,11 +25,18 @@ import androidx.lifecycle.lifecycleScope
 import androidx.work.*
 import com.futo.platformplayer.*
 import com.futo.platformplayer.R
+import com.futo.platformplayer.activities.CaptchaActivity
 import com.futo.platformplayer.activities.IWithResultLauncher
 import com.futo.platformplayer.activities.MainActivity
+import com.futo.platformplayer.api.media.Serializer
+import com.futo.platformplayer.api.media.platforms.js.JSClient
+import com.futo.platformplayer.api.media.platforms.js.internal.JSHttpClient
 import com.futo.platformplayer.background.BackgroundWorker
 import com.futo.platformplayer.casting.StateCasting
 import com.futo.platformplayer.constructs.Event0
+import com.futo.platformplayer.engine.exceptions.ScriptCaptchaRequiredException
+import com.futo.platformplayer.fragment.mainactivity.main.HomeFragment
+import com.futo.platformplayer.fragment.mainactivity.main.SourceDetailFragment
 import com.futo.platformplayer.logging.AndroidLogConsumer
 import com.futo.platformplayer.logging.FileLogConsumer
 import com.futo.platformplayer.logging.LogLevel
@@ -38,7 +45,10 @@ import com.futo.platformplayer.receivers.AudioNoisyReceiver
 import com.futo.platformplayer.services.DownloadService
 import com.futo.platformplayer.stores.FragmentedStorage
 import com.futo.platformplayer.stores.v2.ManagedStore
+import com.stripe.android.core.utils.encodeToJson
 import kotlinx.coroutines.*
+import kotlinx.serialization.encodeToString
+import kotlinx.serialization.json.Json
 import java.io.File
 import java.time.OffsetDateTime
 import java.util.*
@@ -85,7 +95,7 @@ class StateApp {
                     onChanged?.invoke(getExternalGeneralDirectory(context));
                 }
                 else
-                    StateApp.instance.scopeOrNull?.launch(Dispatchers.Main) {
+                    scopeOrNull?.launch(Dispatchers.Main) {
                         UIDialogs.toast("Failed to gain access to\n [${it?.lastPathSegment}]");
                     };
             };
@@ -97,10 +107,14 @@ class StateApp {
         return null;
     }
     fun changeExternalDownloadDirectory(context: IWithResultLauncher, onChanged: ((DocumentFile?)->Unit)? = null) {
+
+        scopeOrNull?.launch(Dispatchers.Main) {
+            UIDialogs.toast("External download directory not yet used by export (WIP)");
+        };
         if(context is Context)
             requestDirectoryAccess(context, "Download Exports", "This directory is used to export downloads to for external usage.", null) {
                 if(it != null)
-                    context.contentResolver.takePersistableUriPermission(it, Intent.FLAG_GRANT_PERSISTABLE_URI_PERMISSION.or(Intent.FLAG_GRANT_WRITE_URI_PERMISSION.or(Intent.FLAG_GRANT_READ_URI_PERMISSION)));
+                    context.contentResolver.takePersistableUriPermission(it, Intent.FLAG_GRANT_WRITE_URI_PERMISSION.or(Intent.FLAG_GRANT_READ_URI_PERMISSION));
                 if(it != null && isValidStorageUri(context, it)) {
                     Logger.i(TAG, "Changed external download directory: ${it}");
                     Settings.instance.storage.storage_general = it.toString();
@@ -419,9 +433,19 @@ class StateApp {
             StatePlaylists.instance.toMigrateCheck()
         ).flatten(), 0);
 
-        scope.launch {
-            delay(5000);
-            StateSubscriptions.instance.updateSubscriptionFeed(scope, false);
+        if(Settings.instance.subscriptions.fetchOnAppBoot) {
+            scope.launch(Dispatchers.IO) {
+                val subRequestCounts = StateSubscriptions.instance.getSubscriptionRequestCount();
+                val reqCountStr = subRequestCounts.map { "    ${it.key.config.name}: ${it.value}/${it.key.config.subscriptionRateLimit}" }.joinToString("\n");
+                val isRateLimitReached = !subRequestCounts.any { clientCount -> clientCount.key.config.subscriptionRateLimit?.let { rateLimit -> clientCount.value > rateLimit } == true };
+                if (isRateLimitReached) {
+                    Logger.w(TAG, "Subscriptions request on boot, request counts:\n${reqCountStr}");
+                    delay(5000);
+                    StateSubscriptions.instance.updateSubscriptionFeed(scope, false);
+                }
+                else
+                    Logger.w(TAG, "Too many subscription requests required:\n${reqCountStr}");
+            }
         }
 
         val interval = Settings.instance.subscriptions.getSubscriptionsBackgroundIntervalMinutes();
@@ -637,6 +661,33 @@ class StateApp {
         }
     }
 
+    private var hasCaptchaDialog = false;
+    fun handleCaptchaException(client: JSClient, exception: ScriptCaptchaRequiredException) {
+        Logger.w(HomeFragment.TAG, "[${client.name}] Plugin captcha required.", exception);
+
+        scopeOrNull?.launch(Dispatchers.Main) {
+            if(hasCaptchaDialog)
+                return@launch;
+            hasCaptchaDialog = true;
+            UIDialogs.showConfirmationDialog(context, "Captcha required\nPlugin [${client.config.name}]", {
+                CaptchaActivity.showCaptcha(context, client.config, exception.url, exception.body) {
+                    hasCaptchaDialog = false;
+                    StatePlugins.instance.setPluginCaptcha(client.config.id, it);
+                    scopeOrNull?.launch(Dispatchers.IO) {
+                        try {
+                            StatePlatform.instance.reloadClient(context, client.config.id);
+                        } catch (e: Throwable) {
+                            Logger.e(SourceDetailFragment.TAG, "Failed to reload client.", e)
+                            return@launch;
+                        }
+                    }
+                }
+            }, {
+                hasCaptchaDialog = false;
+            })
+        }
+    }
+
     companion object {
         private val TAG = "StateApp";
         @SuppressLint("StaticFieldLeak") //This is only alive while MainActivity is alive

app/src/main/java/com/futo/platformplayer/states/StatePlatform.kt

@@ -172,7 +172,11 @@ class StatePlatform {
                     _icons[plugin.config.id] = StatePlugins.instance.getPluginIconOrNull(plugin.config.id) ?:
                             ImageVariable(plugin.config.absoluteIconUrl, null);
 
-                    _availableClients.add(JSClient(context, plugin));
+                    val client = JSClient(context, plugin);
+                    client.onCaptchaException.subscribe { client, ex ->
+                        StateApp.instance.handleCaptchaException(client, ex);
+                    }
+                    _availableClients.add(client);
                 }
 
                 if(_availableClients.distinctBy { it.id }.count() < _availableClients.size)
@@ -287,6 +291,9 @@ class StatePlatform {
                     StatePlugins.instance.getPlugin(id)
                         ?: throw IllegalStateException("Client existed, but plugin config didn't")
                 );
+            newClient.onCaptchaException.subscribe { client, ex ->
+                StateApp.instance.handleCaptchaException(client, ex);
+            }
 
             synchronized(_clientsLock) {
                 if (_enabledClients.contains(client)) {
@@ -618,9 +625,13 @@ class StatePlatform {
     }
 
     fun hasEnabledChannelClient(url : String) : Boolean = getEnabledClients().any { it.isChannelUrl(url) };
-    fun getChannelClient(url : String) : IPlatformClient = getChannelClientOrNull(url)
+    fun getChannelClient(url : String, exclude: List<String>? = null) : IPlatformClient = getChannelClientOrNull(url, exclude)
         ?: throw NoPlatformClientException("No client enabled that supports this channel url (${url})");
-    fun getChannelClientOrNull(url : String) : IPlatformClient? = getEnabledClients().find { it.isChannelUrl(url) };
+    fun getChannelClientOrNull(url : String, exclude: List<String>? = null) : IPlatformClient? =
+        if(exclude == null)
+            getEnabledClients().find { it.isChannelUrl(url) }
+        else
+            getEnabledClients().find { !exclude.contains(it.id) && it.isChannelUrl(url) };
 
     fun getChannel(url: String, updateSubscriptions: Boolean = true): Deferred<IPlatformChannel>  {
         Logger.i(TAG, "Platform - getChannel");
@@ -631,9 +642,9 @@ class StatePlatform {
             return _scope.async { getChannelLive(url, updateSubscriptions) };
     }
 
-    fun getChannelContent(channelUrl: String, isSubscriptionOptimized: Boolean = false, usePooledClients: Int = 0): IPager<IPlatformContent> {
+    fun getChannelContent(channelUrl: String, isSubscriptionOptimized: Boolean = false, usePooledClients: Int = 0, ignorePlugins: List<String>? = null): IPager<IPlatformContent> {
         Logger.i(TAG, "Platform - getChannelVideos");
-        val baseClient = getChannelClient(channelUrl);
+        val baseClient = getChannelClient(channelUrl, ignorePlugins);
         val clientCapabilities = baseClient.getChannelCapabilities();
 
         val client = if(usePooledClients > 1)
@@ -653,19 +664,24 @@ class StatePlatform {
                 toQuery.add(ResultCapabilities.TYPE_STREAMS);
             if(clientCapabilities.hasType(ResultCapabilities.TYPE_LIVE))
                 toQuery.add(ResultCapabilities.TYPE_LIVE);
+            if(clientCapabilities.hasType(ResultCapabilities.TYPE_POSTS))
+                toQuery.add(ResultCapabilities.TYPE_POSTS);
 
             if(isSubscriptionOptimized) {
                 val sub = StateSubscriptions.instance.getSubscription(channelUrl);
                 if(sub != null) {
-                    val daysSinceLiveStream = sub.lastLiveStream.getNowDiffDays()
-                    if(daysSinceLiveStream > 7) {
-                        Logger.i(TAG, "Subscription [${channelUrl}] Last livestream > 7 days, skipping live streams [${daysSinceLiveStream} days ago]");
+                    if(!sub.shouldFetchStreams()) {
+                        Logger.i(TAG, "Subscription [${sub.channel.name}:${channelUrl}] Last livestream > 7 days, skipping live streams [${sub.lastLiveStream.getNowDiffDays()} days ago]");
                         toQuery.remove(ResultCapabilities.TYPE_LIVE);
                     }
-                    if(daysSinceLiveStream > 14) {
-                        Logger.i(TAG, "Subscription [${channelUrl}] Last livestream > 15 days, skipping streams [${daysSinceLiveStream} days ago]");
+                    if(!sub.shouldFetchLiveStreams()) {
+                        Logger.i(TAG, "Subscription [${sub.channel.name}:${channelUrl}] Last livestream > 15 days, skipping streams [${sub.lastLiveStream.getNowDiffDays()} days ago]");
                         toQuery.remove(ResultCapabilities.TYPE_STREAMS);
                     }
+                    if(!sub.shouldFetchPosts()) {
+                        Logger.i(TAG, "Subscription [${sub.channel.name}:${channelUrl}] Last livestream > 5 days, skipping posts [${sub.lastPost.getNowDiffDays()} days ago]");
+                        toQuery.remove(ResultCapabilities.TYPE_POSTS);
+                    }
                 }
             }
 

app/src/main/java/com/futo/platformplayer/states/StatePlaylists.kt

@@ -5,6 +5,7 @@ import android.net.Uri
 import androidx.core.content.FileProvider
 import com.futo.platformplayer.R
 import com.futo.platformplayer.api.media.PlatformID
+import com.futo.platformplayer.api.media.exceptions.NoPlatformClientException
 import com.futo.platformplayer.api.media.models.channels.IPlatformChannel
 import com.futo.platformplayer.api.media.models.contents.IPlatformContent
 import com.futo.platformplayer.api.media.models.video.IPlatformVideo
@@ -265,6 +266,12 @@ class StatePlaylists {
                     builder.messages.add("${name}:[${it}] is no longer available");
                     return@map null;
                 }
+                catch(ex: NoPlatformClientException) {
+                    throw ReconstructionException(name, "No source enabled for [${it}]", ex);
+                    //TODO: Propagate this to dialog, and then back, allowing users to enable plugins...
+                    //builder.messages.add("No source enabled for [${it}]");
+                    //return@map null;
+                }
                 catch(ex: Throwable) {
                     throw ReconstructionException(name, "${name}:[${it}] ${ex.message}", ex);
                 }

app/src/main/java/com/futo/platformplayer/states/StatePlugins.kt

@@ -6,6 +6,7 @@ import com.futo.platformplayer.UIDialogs
 import com.futo.platformplayer.api.http.ManagedHttpClient
 import com.futo.platformplayer.api.media.platforms.js.JSClient
 import com.futo.platformplayer.api.media.platforms.js.SourceAuth
+import com.futo.platformplayer.api.media.platforms.js.SourceCaptchaData
 import com.futo.platformplayer.api.media.platforms.js.SourcePluginConfig
 import com.futo.platformplayer.api.media.platforms.js.SourcePluginDescriptor
 import com.futo.platformplayer.logging.Logger
@@ -372,7 +373,7 @@ class StatePlugins {
             if(icon != null)
                 iconsDir.saveIconBinary(config.id, icon);
 
-            _plugins.save(SourcePluginDescriptor(config, null, flags));
+            _plugins.save(SourcePluginDescriptor(config, null, null, flags));
             return null;
         }
         catch(ex: Throwable) {
@@ -407,6 +408,18 @@ class StatePlugins {
         }
     }
 
+    fun setPluginCaptcha(id: String, captcha: SourceCaptchaData?) {
+        if(id == StateDeveloper.DEV_ID) {
+            StatePlatform.instance.getDevClient()?.let {
+                it.setCaptcha(captcha);
+            };
+            return;
+        }
+
+        val descriptor = getPlugin(id) ?: throw IllegalArgumentException("Plugin [${id}] does not exist");
+        descriptor.updateCaptcha(captcha);
+        _plugins.save(descriptor);
+    }
     fun setPluginAuth(id: String, auth: SourceAuth?) {
         if(id == StateDeveloper.DEV_ID) {
             StatePlatform.instance.getDevClient()?.let {

app/src/main/java/com/futo/platformplayer/states/StatePolycentric.kt

@@ -126,7 +126,7 @@ class StatePolycentric {
         }
     }
 
-    fun getChannelContent(profile: PolycentricProfile, isSubscriptionOptimized: Boolean = false, channelConcurrency: Int = -1): IPager<IPlatformContent> {
+    fun getChannelContent(profile: PolycentricProfile, isSubscriptionOptimized: Boolean = false, channelConcurrency: Int = -1, ignorePlugins: List<String>? = null): IPager<IPlatformContent> {
         //TODO: Currently abusing subscription concurrency for parallelism
         val concurrency = if (channelConcurrency == -1) Settings.instance.subscriptions.getSubscriptionsConcurrency() else channelConcurrency;
         val pagers = profile.ownedClaims.groupBy { it.claim.claimType }.mapNotNull {
@@ -138,7 +138,7 @@ class StatePolycentric {
                 return@mapNotNull null;
             }
 
-            return@mapNotNull StatePlatform.instance.getChannelContent(url, isSubscriptionOptimized, concurrency);
+            return@mapNotNull StatePlatform.instance.getChannelContent(url, isSubscriptionOptimized, concurrency, ignorePlugins);
         }.toTypedArray();
 
         val pager = MultiChronoContentPager(pagers);
@@ -302,7 +302,7 @@ class StatePolycentric {
                         thumbnail =  imageBundle?.selectBestImage(dp_25 * dp_25)?.let { img -> img.toURLInfoSystemLinkUrl(ev.system.toProto(), img.process, listOf(PolycentricCache.SERVER)) },
                         subscribers = null
                     ),
-                    msg = post.content,
+                    msg = if (post.content.count() > PolycentricPlatformComment.MAX_COMMENT_SIZE) post.content.substring(0, PolycentricPlatformComment.MAX_COMMENT_SIZE) else post.content,
                     rating = RatingLikeDislikes(likes, dislikes),
                     date = if (unixMilliseconds != null) Instant.ofEpochMilli(unixMilliseconds).atOffset(ZoneOffset.UTC) else OffsetDateTime.MIN,
                     replyCount = replies.toInt(),

app/src/main/java/com/futo/platformplayer/states/StateSubscriptions.kt

@@ -2,9 +2,12 @@ package com.futo.platformplayer.states
 
 import com.futo.platformplayer.Settings
 import com.futo.platformplayer.UIDialogs
+import com.futo.platformplayer.api.media.models.ResultCapabilities
 import com.futo.platformplayer.api.media.models.channels.IPlatformChannel
 import com.futo.platformplayer.api.media.models.channels.SerializedChannel
 import com.futo.platformplayer.api.media.models.contents.IPlatformContent
+import com.futo.platformplayer.api.media.platforms.js.JSClient
+import com.futo.platformplayer.api.media.platforms.js.SourcePluginConfig
 import com.futo.platformplayer.api.media.structures.*
 import com.futo.platformplayer.api.media.structures.ReusablePager.Companion.asReusable
 import com.futo.platformplayer.cache.ChannelContentCache
@@ -12,9 +15,11 @@ import com.futo.platformplayer.constructs.Event0
 import com.futo.platformplayer.constructs.Event1
 import com.futo.platformplayer.constructs.Event2
 import com.futo.platformplayer.engine.exceptions.PluginException
+import com.futo.platformplayer.engine.exceptions.ScriptCaptchaRequiredException
 import com.futo.platformplayer.exceptions.ChannelException
 import com.futo.platformplayer.findNonRuntimeException
 import com.futo.platformplayer.fragment.mainactivity.main.PolycentricProfile
+import com.futo.platformplayer.getNowDiffDays
 import com.futo.platformplayer.logging.Logger
 import com.futo.platformplayer.models.Subscription
 import com.futo.platformplayer.polycentric.PolycentricCache
@@ -216,13 +221,38 @@ class StateSubscriptions {
         }
     }
 
-    fun getSubscriptionsFeed(allowFailure: Boolean = false): MultiChronoContentPager {
+    fun getSubscriptionRequestCount(): Map<JSClient, Int> {
+        val subs = getSubscriptions();
+        val pluginReqCounts = mutableMapOf<JSClient, Int>();
+
+        for(sub in subs) {
+            val client = StatePlatform.instance.getChannelClientOrNull(sub.channel.url);
+            if(client !is JSClient)
+                continue;
+
+            val channelCaps = client.getChannelCapabilities();
+            if(!pluginReqCounts.containsKey(client))
+                pluginReqCounts[client] = 1;
+            else
+                pluginReqCounts[client] = pluginReqCounts[client]!! + 1;
+
+            if(channelCaps.hasType(ResultCapabilities.TYPE_STREAMS) && sub.shouldFetchStreams())
+                pluginReqCounts[client] = pluginReqCounts[client]!! + 1;
+            if(channelCaps.hasType(ResultCapabilities.TYPE_LIVE) && sub.shouldFetchLiveStreams())
+                pluginReqCounts[client] = pluginReqCounts[client]!! + 1;
+            if(channelCaps.hasType(ResultCapabilities.TYPE_POSTS) && sub.shouldFetchPosts())
+                pluginReqCounts[client] = pluginReqCounts[client]!! + 1;
+        }
+        return pluginReqCounts;
+    }
+
+    fun getSubscriptionsFeed(allowFailure: Boolean = false): IPager<IPlatformContent> {
         val result = getSubscriptionsFeedWithExceptions(allowFailure, true);
         if(result.second.any())
             throw result.second.first();
         return result.first;
     }
-    fun getSubscriptionsFeedWithExceptions(allowFailure: Boolean = false, withCacheFallback: Boolean = false, cacheScope: CoroutineScope? = null, onProgress: ((Int, Int)->Unit)? = null, onNewCacheHit: ((Subscription, IPlatformContent)->Unit)? = null): Pair<MultiChronoContentPager, List<Throwable>> {
+    fun getSubscriptionsFeedWithExceptions(allowFailure: Boolean = false, withCacheFallback: Boolean = false, cacheScope: CoroutineScope? = null, onProgress: ((Int, Int)->Unit)? = null, onNewCacheHit: ((Subscription, IPlatformContent)->Unit)? = null): Pair<IPager<IPlatformContent>, List<Throwable>> {
         val subsPager: Array<IPager<IPlatformContent>>;
         val exs: ArrayList<Throwable> = arrayListOf();
 
@@ -230,8 +260,11 @@ class StateSubscriptions {
         var finished = 0;
         val exceptionMap: HashMap<Subscription, Throwable> = hashMapOf();
         val concurrency = Settings.instance.subscriptions.getSubscriptionsConcurrency();
+        val failedPlugins = arrayListOf<String>();
         for (sub in getSubscriptions().filter { StatePlatform.instance.hasEnabledChannelClient(it.channel.url) }) {
             tasks.add(_subscriptionsPool.submit<Pair<Subscription, IPager<IPlatformContent>?>> {
+                val toIgnore = synchronized(failedPlugins){ failedPlugins.toList() };
+
                 var polycentricProfile : PolycentricCache.CachedPolycentricProfile? = null;
                 val getProfileTime = measureTimeMillis {
                     try {
@@ -258,9 +291,9 @@ class StateSubscriptions {
                     val time = measureTimeMillis {
                         val profile = polycentricProfile?.profile
                         pager = if (profile != null)
-                            StatePolycentric.instance.getChannelContent(profile, true, concurrency)
+                            StatePolycentric.instance.getChannelContent(profile, true, concurrency, toIgnore)
                         else
-                            StatePlatform.instance.getChannelContent(sub.channel.url, true, concurrency);
+                            StatePlatform.instance.getChannelContent(sub.channel.url, true, concurrency, toIgnore);
 
                         if (cacheScope != null)
                             pager = ChannelContentCache.cachePagerResults(cacheScope, pager) {
@@ -276,12 +309,22 @@ class StateSubscriptions {
                     );
                 }
                 catch(ex: Throwable) {
+                    Logger.e(TAG, "Subscription [${sub.channel.name}] failed", ex);
                     finished++;
                     onProgress?.invoke(finished, tasks.size);
                     val channelEx = ChannelException(sub.channel, ex);
                     synchronized(exceptionMap) {
                         exceptionMap.put(sub, channelEx);
                     }
+                    if(ex is ScriptCaptchaRequiredException) {
+                        synchronized(failedPlugins) {
+                            //Fail all subscription calls to plugin if it has a captcha issue
+                            if(ex.config is SourcePluginConfig && !failedPlugins.contains(ex.config.id)) {
+                                Logger.w(TAG, "Subscriptions fetch ignoring plugin [${ex.config.name}] due to Captcha");
+                                failedPlugins.add(ex.config.id);
+                            }
+                        }
+                    }
                     if(!withCacheFallback)
                         throw channelEx;
                     else {
@@ -327,9 +370,10 @@ class StateSubscriptions {
             throw exs.first();
 
         Logger.i(TAG, "Subscription pager with ${subsPager.size} channels");
-        val pager = MultiChronoContentPager(subsPager, allowFailure);
+        val pager = MultiChronoContentPager(subsPager, allowFailure, 15);
         pager.initialize();
-        return Pair(pager, exs);
+        //return Pair(pager, exs);
+        return Pair(DedupContentPager(pager), exs);
     }
 
     //New Migration

app/src/main/java/com/futo/platformplayer/states/StateTelemetry.kt

@@ -37,14 +37,13 @@ class StateTelemetry {
                     BuildConfig.BUILD_TYPE,
                     BuildConfig.DEBUG,
                     BuildConfig.IS_UNSTABLE_BUILD,
-                    Instant.now().epochSecond,
                     Build.BRAND,
                     Build.MANUFACTURER,
                     Build.MODEL
                 );
 
                 val headers = hashMapOf(
-                    "Content-Type" to "text/plain"
+                    "Content-Type" to "application/json"
                 );
 
                 val json = Json.encodeToString(telemetry);

app/src/main/java/com/futo/platformplayer/stores/v2/ManagedStore.kt

@@ -118,6 +118,7 @@ class ManagedStore<T>{
         val builder = ReconstructStore.Builder();
 
         for (recon in items) {
+            onProgress?.invoke(0, total);
             //Retry once
             for (i in 0 .. 1) {
                 try {

app/src/main/java/com/futo/platformplayer/views/adapters/CommentViewHolder.kt

@@ -6,6 +6,7 @@ import android.view.ViewGroup
 import android.widget.ImageView
 import android.widget.LinearLayout
 import android.widget.TextView
+import androidx.constraintlayout.widget.ConstraintLayout
 import androidx.recyclerview.widget.RecyclerView.ViewHolder
 import com.futo.platformplayer.*
 import com.futo.platformplayer.api.media.models.comments.IPlatformComment
@@ -35,12 +36,14 @@ class CommentViewHolder : ViewHolder {
     private val _buttonReplies: PillButton;
     private val _layoutRating: LinearLayout;
     private val _pillRatingLikesDislikes: PillRatingLikesDislikes;
+    private val _layoutComment: ConstraintLayout;
 
     var onClick = Event1<IPlatformComment>();
     var comment: IPlatformComment? = null
         private set;
 
     constructor(viewGroup: ViewGroup) : super(LayoutInflater.from(viewGroup.context).inflate(R.layout.list_comment, viewGroup, false)) {
+        _layoutComment = itemView.findViewById(R.id.layout_comment);
         _creatorThumbnail = itemView.findViewById(R.id.image_thumbnail);
         _textAuthor = itemView.findViewById(R.id.text_author);
         _textMetadata = itemView.findViewById(R.id.text_metadata);
@@ -53,29 +56,31 @@ class CommentViewHolder : ViewHolder {
         _layoutRating = itemView.findViewById(R.id.layout_rating);
         _pillRatingLikesDislikes = itemView.findViewById(R.id.rating);
 
-        _pillRatingLikesDislikes.onLikeDislikeUpdated.subscribe { processHandle, hasLiked, hasDisliked ->
+        _pillRatingLikesDislikes.onLikeDislikeUpdated.subscribe { args ->
             val c = comment
             if (c !is PolycentricPlatformComment) {
                 throw Exception("Not implemented for non polycentric comments")
             }
 
-            if (hasLiked) {
-                processHandle.opinion(c.reference, Opinion.like);
-            } else if (hasDisliked) {
-                processHandle.opinion(c.reference, Opinion.dislike);
+            if (args.hasLiked) {
+                args.processHandle.opinion(c.reference, Opinion.like);
+            } else if (args.hasDisliked) {
+                args.processHandle.opinion(c.reference, Opinion.dislike);
             } else {
-                processHandle.opinion(c.reference, Opinion.neutral);
+                args.processHandle.opinion(c.reference, Opinion.neutral);
             }
 
+            _layoutComment.alpha = if (args.dislikes > 2 && args.dislikes / (args.likes + args.dislikes) >= 0.7) 0.5f else 1.0f;
+
             StateApp.instance.scopeOrNull?.launch(Dispatchers.IO) {
                 try {
-                    processHandle.fullyBackfillServers();
+                    args.processHandle.fullyBackfillServers();
                 } catch (e: Throwable) {
                     Logger.e(TAG, "Failed to backfill servers.", e)
                 }
             }
 
-            StatePolycentric.instance.updateLikeMap(c.reference, hasLiked, hasDisliked)
+            StatePolycentric.instance.updateLikeMap(c.reference, args.hasLiked, args.hasDisliked)
         };
 
         _buttonReplies.onClick.subscribe {
@@ -98,6 +103,13 @@ class CommentViewHolder : ViewHolder {
             _textMetadata.visibility = View.GONE;
         }
 
+        val rating = comment.rating;
+        if (rating is RatingLikeDislikes) {
+            _layoutComment.alpha = if (rating.dislikes > 0 && rating.dislikes / (rating.likes + rating.dislikes) >= 0.7) 0.5f else 1.0f;
+        } else {
+            _layoutComment.alpha = 1.0f;
+        }
+
         _textBody.text = comment.message.fixHtmlLinks();
 
         if (readonly) {

app/src/main/java/com/futo/platformplayer/views/pills/PillRatingLikesDislikes.kt

@@ -12,11 +12,20 @@ import com.futo.platformplayer.R
 import com.futo.platformplayer.api.media.models.ratings.IRating
 import com.futo.platformplayer.api.media.models.ratings.RatingLikeDislikes
 import com.futo.platformplayer.api.media.models.ratings.RatingLikes
+import com.futo.platformplayer.constructs.Event1
 import com.futo.platformplayer.constructs.Event3
 import com.futo.platformplayer.states.StatePolycentric
 import com.futo.platformplayer.toHumanNumber
 import com.futo.polycentric.core.ProcessHandle
 
+data class OnLikeDislikeUpdatedArgs(
+    val processHandle: ProcessHandle,
+    val likes: Long,
+    val hasLiked: Boolean,
+    val dislikes: Long,
+    val hasDisliked: Boolean,
+);
+
 class PillRatingLikesDislikes : LinearLayout {
     private val _textLikes: TextView;
     private val _textDislikes: TextView;
@@ -29,7 +38,7 @@ class PillRatingLikesDislikes : LinearLayout {
     private var _dislikes = 0L;
     private var _hasDisliked = false;
 
-    val onLikeDislikeUpdated = Event3<ProcessHandle, Boolean, Boolean>();
+    val onLikeDislikeUpdated = Event1<OnLikeDislikeUpdatedArgs>();
 
     constructor(context : Context, attrs : AttributeSet?) : super(context, attrs) {
         LayoutInflater.from(context).inflate(R.layout.rating_likesdislikes, this, true);
@@ -76,7 +85,7 @@ class PillRatingLikesDislikes : LinearLayout {
 
         _textLikes.text = _likes.toHumanNumber();
         updateColors();
-        onLikeDislikeUpdated.emit(processHandle, _hasLiked, _hasDisliked);
+        onLikeDislikeUpdated.emit(OnLikeDislikeUpdatedArgs(processHandle, _likes, _hasLiked, _dislikes, _hasDisliked));
     }
 
     fun dislike(processHandle: ProcessHandle) {
@@ -96,7 +105,7 @@ class PillRatingLikesDislikes : LinearLayout {
 
         _textDislikes.text = _dislikes.toHumanNumber();
         updateColors();
-        onLikeDislikeUpdated.emit(processHandle, _hasLiked, _hasDisliked);
+        onLikeDislikeUpdated.emit(OnLikeDislikeUpdatedArgs(processHandle, _likes, _hasLiked, _dislikes, _hasDisliked));
     }
 
     private fun updateColors() {

app/src/main/res/layout/activity_captcha.xml

@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="utf-8"?>
+<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent"
+    android:orientation="vertical"
+    android:background="@color/black">
+
+    <LinearLayout
+        android:layout_width="match_parent"
+        android:layout_height="wrap_content"
+        android:orientation="horizontal">
+
+        <TextView
+            android:layout_width="0dp"
+            android:layout_weight="1"
+            android:layout_height="wrap_content"
+            android:layout_margin="4dp"
+            android:layout_gravity="center_vertical"
+            android:text="Please enter the captcha and close when finished" />
+
+        <Button
+            android:id="@+id/button_close"
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"
+            android:layout_gravity="end"
+            android:layout_marginEnd="6dp"
+            android:text="CLOSE" />
+    </LinearLayout>
+
+
+    <WebView
+        android:id="@+id/web_view"
+        android:layout_width="match_parent"
+        android:layout_height="match_parent" />
+</LinearLayout>

app/src/main/res/layout/dialog_comment.xml

@@ -35,6 +35,25 @@
                 android:gravity="center"
                 android:layout_marginTop="12dp">
 
+                <TextView
+                    android:id="@+id/character_count"
+                    android:layout_width="wrap_content"
+                    android:layout_height="wrap_content"
+                    android:text="0"
+                    android:textSize="14dp"
+                    android:textColor="@color/white"
+                    android:fontFamily="@font/inter_regular"
+                    android:layout_marginStart="24dp"/>
+
+                <TextView
+                    android:id="@+id/character_count_max"
+                    android:layout_width="wrap_content"
+                    android:layout_height="wrap_content"
+                    android:text="/2000"
+                    android:textSize="14dp"
+                    android:textColor="@color/white"
+                    android:fontFamily="@font/inter_regular" />
+
                 <Space
                     android:layout_width="0dp"
                     android:layout_height="wrap_content"

app/src/main/res/layout/list_comment.xml

@@ -62,6 +62,7 @@
         android:isScrollContainer="false"
         android:textColor="#CCCCCC"
         android:textSize="13sp"
+        android:maxLines="100"
         app:layout_constraintTop_toBottomOf="@id/text_metadata"
         app:layout_constraintLeft_toRightOf="@id/image_thumbnail"
         app:layout_constraintRight_toRightOf="parent"