images update, sync since all time option, qdb write location fixes, lastUsersSync fixes

src/modules/api/submodules/p2p.ts

@@ -176,7 +176,11 @@ export function mergeQdbs(
         const qdbMergeResult = mergeResults.find(
             (mergeRes) => mergeRes.localQdbIndex === qdb.index
         )
-        if (qdbMergeResult) {
+        if (
+            qdbMergeResult &&
+            (qdbMergeResult.newData.length > 0 ||
+                qdbMergeResult.newSubjects.length > 0)
+        ) {
             const mergedQdb = {
                 ...qdb,
                 data: mergeSubjects(
@@ -273,23 +277,34 @@ function writeNewData(
     const qdbsToWrite = [...changedQuestionDbs, ...newQuestionDbs]
     const existingQdbs = utils.ReadJSON<DataFile[]>(dbsFilePath)
 
-    const qdbDataToWrite = qdbsToWrite.reduce((acc, qdb) => {
+    const qdbDataToWrite = qdbsToWrite
-        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        .reduce((acc, qdb) => {
-        const { data, index, ...qdbData } = qdb
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
-        const existingQdbData = acc.find((data) => {
+            const { data, index, ...qdbData } = qdb
-            return data.name === qdbData.name
+            const existingQdbData = acc.find((data) => {
+                return data.name === qdbData.name
+            })
+            if (!existingQdbData) {
+                return [...acc, qdbData]
+            } else {
+                return acc
+            }
+        }, existingQdbs)
+        .map((qdb) => {
+            if (qdb.path.includes(publicDir)) {
+                return {
+                    ...qdb,
+                    path: qdb.path.replace(publicDir, ''),
+                }
+            } else {
+                return qdb
+            }
         })
-        if (!existingQdbData) {
-            return [...acc, qdbData]
-        } else {
-            return acc
-        }
-    }, existingQdbs)
 
     utils.WriteFile(JSON.stringify(qdbDataToWrite, null, 2), dbsFilePath)
     qdbsToWrite.forEach((qdb) => {
         try {
-            writeData(qdb.data, publicDir + qdb.path)
+            writeData(qdb.data, qdb.path)
         } catch (e) {
             logger.Log(`Error writing ${qdb.name} qdb to file!`, 'redbg')
             console.error(e)
@@ -321,23 +336,23 @@ function setupQuestionsForMerge(qdb: QuestionDb, peer: PeerInfo) {
 }
 
 async function authAndGetNewData({
-    peers,
     peer,
     selfInfo,
+    allTime,
 }: {
-    peers: PeerInfo[]
     peer: PeerInfo
     selfInfo: PeerInfo
+    allTime?: boolean
 }): Promise<GetResult<SyncDataRes & { peer: PeerInfo }>> {
     let sessionCookie = peer.sessionCookie
-    const lastSyncWithPeer = peer.lastSync || 0
+    const lastSyncWithPeer = allTime ? 0 : peer.lastSync || 0
-    const lastUsersSyncWithPeer = peer.lastUsersSync || 0
+    const lastUsersSyncWithPeer = allTime ? 0 : peer.lastUsersSync || 0
 
     if (!sessionCookie) {
         const loginResult = await loginToPeer(peer)
         if (typeof loginResult === 'string') {
             sessionCookie = loginResult
-            updatePeersFile(peers, { ...peer, sessionCookie: loginResult })
+            updatePeersFile({ ...peer, sessionCookie: loginResult })
         } else {
             return {
                 error: loginResult,
@@ -375,7 +390,7 @@ async function authAndGetNewData({
         const loginResult = await loginToPeer(peer)
         if (typeof loginResult === 'string') {
             sessionCookie = loginResult
-            updatePeersFile(peers, { ...peer, sessionCookie: loginResult })
+            updatePeersFile({ ...peer, sessionCookie: loginResult })
         } else {
             return {
                 error: loginResult,
@@ -559,7 +574,7 @@ function setup(data: SubmoduleData): Submodule {
             return {
                 ...qdb,
                 index: availableIndexes[i],
-                path: `questionDbs/${qdb.name}.json`,
+                path: `${publicDir}questionDbs/${qdb.name}.json`,
             }
         })
     }
@@ -595,7 +610,13 @@ function setup(data: SubmoduleData): Submodule {
         }
     }
 
-    async function syncData(usersOnly?: boolean) {
+    async function syncData({
+        usersOnly,
+        allTime,
+    }: {
+        usersOnly: boolean
+        allTime: boolean
+    }) {
         if (peers.length === 0) {
             logger.Log(
                 `There are no peers specified in ${paths.peersFile}, aborting sync`,
@@ -611,21 +632,27 @@ function setup(data: SubmoduleData): Submodule {
                 peers.length
             }${logger.C()} peers`
         )
+
         if (usersOnly) {
             logger.Log(`\tSyncing users only!`, 'yellowbg')
         }
+        if (allTime) {
+            logger.Log(`\tSyncing since all time!`, 'yellowbg')
+        }
 
         const lastSync = selfInfo.lastSync
         logger.Log(
-            `\tLast sync date: ${logger.C('blue')}${new Date(
+            `\tLast sync date: ${logger.C('blue')}${
-                lastSync
+                lastSync ? new Date(lastSync).toLocaleString() : 'never'
-            ).toLocaleString()}${logger.C()}`
+            }${logger.C()}`
         )
         const syncStart = new Date().getTime()
         const lastSyncInfos = peers.map((peer) => {
             return [
                 peerToString(peer),
-                new Date(peer.lastSync).toLocaleString(),
+                peer.lastSync
+                    ? new Date(peer.lastSync).toLocaleString()
+                    : 'never',
             ]
         })
         logger.Log(`\tLast sync with peers:`)
@@ -636,9 +663,9 @@ function setup(data: SubmoduleData): Submodule {
 
         const requests = peers.map((peer) => {
             return authAndGetNewData({
-                peers: peers,
                 peer: peer,
                 selfInfo: selfInfo,
+                allTime: allTime,
             })
         })
 
@@ -688,7 +715,7 @@ function setup(data: SubmoduleData): Submodule {
             if (questionCount > 0) {
                 resultDataWithoutEmptyDbs.push(res)
             } else if (!usersOnly) {
-                updatePeersFile(peers, {
+                updatePeersFile({
                     ...res.peer,
                     lastSync: syncStart,
                 })
@@ -775,7 +802,7 @@ function setup(data: SubmoduleData): Submodule {
                     resultsCount[peerToString(res.peer)] = {
                         newUsers: addedUserCount,
                     }
-                    updatePeersFile(peers, {
+                    updatePeersFile({
                         ...res.peer,
                         lastUsersSync: syncStart,
                     })
@@ -894,7 +921,7 @@ function setup(data: SubmoduleData): Submodule {
                 newQuestions: newQuestionCount,
             }
             // Processing result data is successfull
-            updatePeersFile(peers, {
+            updatePeersFile({
                 ...peer,
                 lastSync: syncStart,
             })
@@ -1092,9 +1119,9 @@ function setup(data: SubmoduleData): Submodule {
             }
         }
 
-        const usersSinceDate = Number.isNaN(since)
+        const usersSinceDate = usersSince
-            ? 'all time'
+            ? new Date(since).toLocaleString()
-            : new Date(since).toLocaleString()
+            : 'all time'
 
         if (usersOnly) {
             logger.Log('Sending users only!', 'yellowbg')
@@ -1108,9 +1135,10 @@ function setup(data: SubmoduleData): Submodule {
                 )}${sentUsers}${logger.C()}`
             )
         } else {
-            const dateToLog = Number.isNaN(since)
+            console.log(since)
-                ? 'all time'
+            const dateToLog = since
-                : new Date(since).toLocaleString()
+                ? new Date(since).toLocaleString()
+                : 'all time'
 
             logger.Log(
                 `\tSending new data to ${logger.C(
@@ -1141,6 +1169,7 @@ function setup(data: SubmoduleData): Submodule {
     app.get('/syncp2pdata', (req: Request, res: Response) => {
         logger.LogReq(req)
         const usersOnly = !!req.query.usersOnly
+        const allTime = !!req.query.allTime
         const user = req.session.user
         if (!user || user.id !== 1) {
             res.json({
@@ -1160,7 +1189,7 @@ function setup(data: SubmoduleData): Submodule {
 
         syncInProgress = true
         setPendingJobsAlertCount(5000)
-        syncData(usersOnly)
+        syncData({ usersOnly: usersOnly, allTime: allTime })
             .then((syncResult) => {
                 res.json({
                     msg: 'sync successfull',

src/utils/p2putils.ts

@@ -1,5 +1,5 @@
 import { PeerInfo } from '../types/basicTypes'
-import { paths } from './files'
+import { files, paths, readAndValidateFile } from './files'
 import { parseCookie, post } from './networkUtils'
 import utils from './utils'
 
@@ -14,10 +14,15 @@ export function isPeerSameAs(peer1: PeerInfo, peer2: PeerInfo): boolean {
     return peer1.host === peer2.host && peer1.port === peer2.port
 }
 
-export function updatePeersFile(
+export function updatePeersFile(updatedPeer: PeerInfo): void {
-    peers: PeerInfo[],
+    const newVal = readAndValidateFile<PeerInfo[]>(files.peersFile)
-    updatedPeer: PeerInfo
+    let peers
-): void {
+    if (newVal) {
+        peers = newVal
+    }
+    if (!peers)
+        throw new Error('Peers file was invalid while trying to update it!')
+
     const updatedPeers = peers.map((x) => {
         if (isPeerSameAs(updatedPeer, x)) {
             return {

src/worker/handlers/handleQuestionsToPeers.ts

@@ -98,7 +98,7 @@ export const handleQuestionsToPeers = async (
                 results.loginErrors.push(peer)
                 continue
             }
-            updatePeersFile(peers, { ...peer, sessionCookie: sessionCookie })
+            updatePeersFile({ ...peer, sessionCookie: sessionCookie })
         }
 
         let res = await postData(peer, sessionCookie)
@@ -109,7 +109,7 @@ export const handleQuestionsToPeers = async (
                 results.loginErrors.push(peer)
                 continue
             }
-            updatePeersFile(peers, { ...peer, sessionCookie: sessionCookie })
+            updatePeersFile({ ...peer, sessionCookie: sessionCookie })
             res = await postData(peer, sessionCookie)
         }
 

submodules/moodle-test-userscript

@@ -1 +1 @@
-Subproject commit 9220466561fce5a42111552a9acdc0d0fa28ad61
+Subproject commit 2ee9731fdf350bbddb54b0ce34b590a5c79fbd03

submodules/qmining-page

@@ -1 +1 @@
-Subproject commit 97e1914bf67f23c2e2dd84df7a4d3021dc2d3382
+Subproject commit 2e12f3275d47334acb97437ecb9f72af4261d334