user only sync and new images

src/modules/api/submodules/p2p.ts

@@ -324,14 +324,14 @@ async function authAndGetNewData({
     peers,
     peer,
     selfInfo,
-    lastSyncWithPeer,
 }: {
     peers: PeerInfo[]
     peer: PeerInfo
     selfInfo: PeerInfo
-    lastSyncWithPeer: number
 }): Promise<GetResult<SyncDataRes & { peer: PeerInfo }>> {
     let sessionCookie = peer.sessionCookie
+    const lastSyncWithPeer = peer.lastSync || 0
+    const lastUsersSyncWithPeer = peer.lastUsersSync || 0
 
     if (!sessionCookie) {
         const loginResult = await loginToPeer(peer)
@@ -358,7 +358,11 @@ async function authAndGetNewData({
                 port: peer.port,
                 path: `/api/getnewdatasince?host=${encodeURIComponent(
                     peerToString(selfInfo)
-                )}${lastSyncWithPeer ? `&since=${lastSyncWithPeer}` : ''}`,
+                )}${lastSyncWithPeer ? `&since=${lastSyncWithPeer}` : ''}${
+                    lastUsersSyncWithPeer
+                        ? `&usersSince=${lastUsersSyncWithPeer}`
+                        : ''
+                }`,
             },
             peer.http
         )
@@ -591,7 +595,7 @@ function setup(data: SubmoduleData): Submodule {
         }
     }
 
-    async function syncData() {
+    async function syncData(usersOnly?: boolean) {
         if (peers.length === 0) {
             logger.Log(
                 `There are no peers specified in ${paths.peersFile}, aborting sync`,
@@ -607,6 +611,9 @@ function setup(data: SubmoduleData): Submodule {
                 peers.length
             }${logger.C()} peers`
         )
+        if (usersOnly) {
+            logger.Log(`\tSyncing users only!`, 'yellowbg')
+        }
 
         const lastSync = selfInfo.lastSync
         logger.Log(
@@ -628,13 +635,10 @@ function setup(data: SubmoduleData): Submodule {
         })
 
         const requests = peers.map((peer) => {
-            const lastSyncWithPeer = peer.lastSync || 0
-
             return authAndGetNewData({
                 peers: peers,
                 peer: peer,
                 selfInfo: selfInfo,
-                lastSyncWithPeer: lastSyncWithPeer,
             })
         })
 
@@ -683,7 +687,7 @@ function setup(data: SubmoduleData): Submodule {
 
             if (questionCount > 0) {
                 resultDataWithoutEmptyDbs.push(res)
-            } else {
+            } else if (!usersOnly) {
                 updatePeersFile(peers, {
                     ...res.peer,
                     lastSync: syncStart,
@@ -748,6 +752,7 @@ function setup(data: SubmoduleData): Submodule {
         try {
             resultData.forEach((res) => {
                 if (res.encryptedUsers) {
+                    let addedUserCount = 0
                     const decryptedUsers: User[] = JSON.parse(
                         decrypt(privateKey, res.encryptedUsers)
                     )
@@ -758,6 +763,7 @@ function setup(data: SubmoduleData): Submodule {
                             pw: remoteUser.pw,
                         })
                         if (localUser.length === 0) {
+                            addedUserCount += 1
                             // FIXME: users will not have consistend id across servers. This may be
                             // harmless, will see
                             dbtools.Insert(userDB, 'users', {
@@ -767,8 +773,12 @@ function setup(data: SubmoduleData): Submodule {
                         }
                     })
                     resultsCount[peerToString(res.peer)] = {
-                        newUsers: decryptedUsers.length,
+                        newUsers: addedUserCount,
                     }
+                    updatePeersFile(peers, {
+                        ...res.peer,
+                        lastUsersSync: syncStart,
+                    })
                 }
             })
         } catch (e) {
@@ -794,6 +804,40 @@ function setup(data: SubmoduleData): Submodule {
             }
         }
 
+        if (usersOnly) {
+            if (Object.keys(resultsCount).length === 0) {
+                logger.Log('No new users received')
+            } else {
+                logger.logTable(
+                    [
+                        ['', 'Users'],
+                        ['Old', oldUserCount],
+                        ...Object.entries(resultsCount).map(([key, result]) => {
+                            return [key, result.newUsers]
+                        }),
+                        ['Added total', newUserCount - oldUserCount],
+                        ['Final', newUserCount],
+                    ],
+                    { colWidth: [15], rowPrefix: '\t' }
+                )
+            }
+            logger.Log(
+                `Sync successfully finished! Synced users only.`,
+                'green'
+            )
+            return {
+                old: {
+                    oldUserCount: oldUserCount,
+                },
+                added: {
+                    totalNewUers: newUserCount - oldUserCount,
+                },
+                final: {
+                    newUserCount: newUserCount,
+                },
+            }
+        }
+
         // -------------------------------------------------------------------------------------------------------
         // backup
         // -------------------------------------------------------------------------------------------------------
@@ -930,6 +974,7 @@ function setup(data: SubmoduleData): Submodule {
                 oldQuestionCount: oldQuestionCount,
             },
             added: {
+                totalNewUers: newUsers,
                 totalNewQdbs: totalNewQdbs,
                 totalNewSubjects: totalNewSubjects,
                 totalNewQuestions: totalNewQuestions,
@@ -962,8 +1007,17 @@ function setup(data: SubmoduleData): Submodule {
         // a getNewDataSince() call per question db
         logger.LogReq(req)
         const since = Number.isNaN(+req.query.since) ? 0 : +req.query.since
+        const usersSince = Number.isNaN(+req.query.usersSince)
+            ? 0
+            : +req.query.usersSince
         const remoteHost = req.query.host
+        const usersOnly = !!req.query.usersOnly
 
+        const result: SyncDataRes = {
+            remoteInfo: getSelfInfo(),
+        }
+
+        if (!usersOnly) {
             const questionDbsWithNewQuestions = Number.isNaN(since)
                 ? getQuestionDbs()
                 : getQuestionDbs()
@@ -974,22 +1028,20 @@ function setup(data: SubmoduleData): Submodule {
                           }
                       })
                       .filter((qdb) => {
-                      const { questionCount: questionCount } = countOfQdb(qdb)
+                          const { questionCount: questionCount } =
+                              countOfQdb(qdb)
                           return questionCount > 0
                       })
 
-        const { subjCount: subjects, questionCount: questions } = countOfQdbs(
+            const { subjCount: subjects, questionCount: questions } =
-            questionDbsWithNewQuestions
+                countOfQdbs(questionDbsWithNewQuestions)
-        )
 
-        const result: SyncDataRes = {
+            result.questionDbs = questionDbsWithNewQuestions
-            questionDbs: questionDbsWithNewQuestions,
+            result.count = {
-            count: {
                 qdbs: questionDbsWithNewQuestions.length,
                 subjects: subjects,
                 questions: questions,
-            },
+            }
-            remoteInfo: getSelfInfo(),
         }
 
         let hostToLog = remoteHost || 'Unknown host'
@@ -1003,7 +1055,7 @@ function setup(data: SubmoduleData): Submodule {
                 const remotePublicKey = remotePeerInfo?.publicKey
                 if (remotePublicKey) {
                     // FIXME: sign data?
-                    const newUsers = getNewUsersSince(since)
+                    const newUsers = getNewUsersSince(usersSince)
                     sentUsers = newUsers.length
                     result.encryptedUsers = encrypt(
                         remotePublicKey,
@@ -1040,6 +1092,22 @@ function setup(data: SubmoduleData): Submodule {
             }
         }
 
+        const usersSinceDate = Number.isNaN(since)
+            ? 'all time'
+            : new Date(since).toLocaleString()
+
+        if (usersOnly) {
+            logger.Log('Sending users only!', 'yellowbg')
+            logger.Log(
+                `\tSending new users to ${logger.C(
+                    'blue'
+                )}${hostToLog}${logger.C()} since  ${logger.C(
+                    'blue'
+                )}${usersSinceDate}${logger.C()}. Sent users: ${logger.C(
+                    'blue'
+                )}${sentUsers}${logger.C()}`
+            )
+        } else {
             const dateToLog = Number.isNaN(since)
                 ? 'all time'
                 : new Date(since).toLocaleString()
@@ -1049,31 +1117,35 @@ function setup(data: SubmoduleData): Submodule {
                     'blue'
                 )}${hostToLog}${logger.C()} since ${logger.C(
                     'blue'
-            )}${dateToLog}${logger.C()} `
+                )}${dateToLog}${logger.C()}, and new users since ${logger.C(
+                    'blue'
+                )}${usersSinceDate}${logger.C()}`
             )
             logger.logTable(
                 [
                     ['Users', 'QDBs', 'Subjs', 'Questions'],
                     [
                         sentUsers,
-                    questionDbsWithNewQuestions.length,
+                        result.questionDbs.length,
-                    subjects,
+                        result.count.subjects,
-                    questions,
+                        result.count.questions,
                     ],
                 ],
                 { rowPrefix: '\t' }
             )
+        }
 
         res.json(result)
     })
 
     app.get('/syncp2pdata', (req: Request, res: Response) => {
         logger.LogReq(req)
+        const usersOnly = !!req.query.usersOnly
         const user = req.session.user
         if (!user || user.id !== 1) {
             res.json({
                 status: 'error',
-                msg: 'only user 1 can call this EP',
+                message: 'only user 1 can call this EP',
             })
             return
         }
@@ -1088,7 +1160,7 @@ function setup(data: SubmoduleData): Submodule {
 
         syncInProgress = true
         setPendingJobsAlertCount(5000)
-        syncData()
+        syncData(usersOnly)
             .then((syncResult) => {
                 res.json({
                     msg: 'sync successfull',

src/types/basicTypes.ts

@@ -177,6 +177,7 @@ export interface PeerInfo {
     pw?: string
     sessionCookie?: string
     lastSync?: number
+    lastUsersSync?: number
     note?: string
     http?: boolean
 }

src/types/typeSchemas.ts

@@ -59,6 +59,7 @@ export const PeerInfoSchema: Schema = {
     ...PeerInfoSchemaBase,
     properties: {
         ...PeerInfoSchemaBase.properties,
+        lastUsersSync: { type: 'number' },
         publicKey: { type: 'string' },
         pw: { type: 'string' },
         sessionCookie: { type: 'string' },

submodules/moodle-test-userscript

@@ -1 +1 @@
-Subproject commit aad2822226c6fc039ee190581b0eaa55f2859dc9
+Subproject commit 898326c3a061e786d9ff8cf437059db34da50504