mirror of
https://gitlab.com/MrFry/mrfrys-node-server
synced 2025-04-01 20:24:18 +02:00
worker file split, sending new questions to peers instantly
This commit is contained in:
mrfry
parent
8c4e184741
commit
252826a081
25 changed files with 1016 additions and 705 deletions
|
|
@ -39,8 +39,8 @@ import {
|
|||
Submodule,
|
||||
} from '../../types/basicTypes'
|
||||
import { loadJSON } from '../../utils/actions'
|
||||
import { initWorkerPool } from '../../utils/workerPool'
|
||||
import { paths } from '../../utils/files'
|
||||
import { initWorkerPool } from '../../worker/workerPool'
|
||||
|
||||
// other paths
|
||||
const moduleName = 'API'
|
||||
|
|
|
|||
|
|
@ -33,7 +33,6 @@ import {
|
|||
} from '../../../types/basicTypes'
|
||||
import utils from '../../../utils/utils'
|
||||
import { backupData, writeData } from '../../../utils/actions'
|
||||
import { WorkerResult } from '../../../utils/classes'
|
||||
import dbtools from '../../../utils/dbtools'
|
||||
import {
|
||||
createKeyPair,
|
||||
|
|
@ -41,11 +40,6 @@ import {
|
|||
encrypt,
|
||||
isKeypairValid,
|
||||
} from '../../../utils/encryption'
|
||||
import {
|
||||
doALongTask,
|
||||
msgAllWorker,
|
||||
setPendingJobsAlertCount,
|
||||
} from '../../../utils/workerPool'
|
||||
import {
|
||||
countOfQdb,
|
||||
countOfQdbs,
|
||||
|
|
@ -54,7 +48,18 @@ import {
|
|||
removeCacheFromQuestion,
|
||||
} from '../../../utils/qdbUtils'
|
||||
import { files, paths, readAndValidateFile } from '../../../utils/files'
|
||||
import { GetResult, get, post } from '../../../utils/networkUtils'
|
||||
import { GetResult, get } from '../../../utils/networkUtils'
|
||||
import {
|
||||
msgAllWorker,
|
||||
queueWork,
|
||||
setPendingJobsAlertCount,
|
||||
} from '../../../worker/workerPool'
|
||||
import { WorkerResult } from '../../../worker/worker'
|
||||
import {
|
||||
loginToPeer,
|
||||
peerToString,
|
||||
updatePeersFile,
|
||||
} from '../../../utils/p2putils'
|
||||
|
||||
interface MergeResult {
|
||||
newData: Subject[]
|
||||
|
|
@ -87,6 +92,7 @@ interface RemotePeerInfo {
|
|||
}
|
||||
|
||||
interface SyncDataRes {
|
||||
result?: string
|
||||
questionDbs?: QuestionDb[]
|
||||
remoteInfo?: RemotePeerInfo
|
||||
encryptedUsers?: string
|
||||
|
|
@ -120,14 +126,6 @@ function updateThirdPartyPeers(
|
|||
)
|
||||
}
|
||||
|
||||
function peerToString(peer: { host: string; port: string | number }) {
|
||||
return `${peer.host}:${peer.port}`
|
||||
}
|
||||
|
||||
function isPeerSameAs(peer1: PeerInfo, peer2: PeerInfo) {
|
||||
return peer1.host === peer2.host && peer1.port === peer2.port
|
||||
}
|
||||
|
||||
export function getNewDataSince(subjects: Subject[], date: number): Subject[] {
|
||||
return subjects
|
||||
.map((subject) => {
|
||||
|
|
@ -323,49 +321,69 @@ function setupQuestionsForMerge(qdb: QuestionDb, peer: PeerInfo) {
|
|||
}
|
||||
|
||||
async function authAndGetNewData({
|
||||
peers,
|
||||
peer,
|
||||
selfInfo,
|
||||
lastSyncWithPeer,
|
||||
lastSync,
|
||||
}: {
|
||||
peers: PeerInfo[]
|
||||
peer: PeerInfo
|
||||
selfInfo: PeerInfo
|
||||
lastSyncWithPeer: number
|
||||
lastSync: number
|
||||
}): Promise<GetResult<SyncDataRes & { peer: PeerInfo }>> {
|
||||
const { data, error, cookies } = await post<{
|
||||
result: string
|
||||
msg: string
|
||||
}>({
|
||||
hostname: peer.host,
|
||||
path: '/api/login',
|
||||
port: peer.port,
|
||||
bodyObject: { pw: peer.pw },
|
||||
http: peer.http,
|
||||
})
|
||||
let sessionCookie = peer.sessionCookie
|
||||
|
||||
if (error || !data || data.result !== 'success') {
|
||||
return {
|
||||
error: data ? new Error(data.msg) : error,
|
||||
data: {
|
||||
peer: peer,
|
||||
},
|
||||
if (!sessionCookie) {
|
||||
const loginResult = await loginToPeer(peer)
|
||||
if (typeof loginResult === 'string') {
|
||||
sessionCookie = loginResult
|
||||
updatePeersFile(peers, { ...peer, sessionCookie: loginResult })
|
||||
} else {
|
||||
return {
|
||||
error: loginResult,
|
||||
data: {
|
||||
peer: peer,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const getRes = await get<SyncDataRes>(
|
||||
{
|
||||
headers: {
|
||||
cookie: cookies.join(),
|
||||
const getSyncData = () => {
|
||||
return get<SyncDataRes>(
|
||||
{
|
||||
headers: {
|
||||
cookies: `sessionID=${sessionCookie}`,
|
||||
},
|
||||
host: peer.host,
|
||||
port: peer.port,
|
||||
path: `/api/getnewdatasince?host=${encodeURIComponent(
|
||||
peerToString(selfInfo)
|
||||
)}${lastSync ? `&since=${lastSyncWithPeer}` : ''}`,
|
||||
},
|
||||
host: peer.host,
|
||||
port: peer.port,
|
||||
path: `/api/getnewdatasince?host=${encodeURIComponent(
|
||||
peerToString(selfInfo)
|
||||
)}${lastSync ? `&since=${lastSyncWithPeer}` : ''}`,
|
||||
},
|
||||
peer.http
|
||||
)
|
||||
peer.http
|
||||
)
|
||||
}
|
||||
|
||||
let getRes = await getSyncData()
|
||||
|
||||
if (getRes.data?.result === 'nouser') {
|
||||
// FIXME: make this more pretty? (duplicate code, see above)
|
||||
const loginResult = await loginToPeer(peer)
|
||||
if (typeof loginResult === 'string') {
|
||||
sessionCookie = loginResult
|
||||
updatePeersFile(peers, { ...peer, sessionCookie: loginResult })
|
||||
} else {
|
||||
return {
|
||||
error: loginResult,
|
||||
data: {
|
||||
peer: peer,
|
||||
},
|
||||
}
|
||||
}
|
||||
getRes = await getSyncData()
|
||||
}
|
||||
|
||||
return { ...getRes, data: { ...getRes.data, peer: peer } }
|
||||
}
|
||||
|
|
@ -385,21 +403,6 @@ function setup(data: SubmoduleData): Submodule {
|
|||
// SETUP
|
||||
// ---------------------------------------------------------------------------------------
|
||||
|
||||
if (!utils.FileExists(paths.peersFile)) {
|
||||
logger.Log(
|
||||
`Warning: peers file was missing, so it was created`,
|
||||
'yellowbg'
|
||||
)
|
||||
utils.CreatePath(paths.peersPath)
|
||||
utils.WriteFile('[]', paths.peersFile)
|
||||
}
|
||||
|
||||
if (!utils.FileExists(paths.selfInfoFile)) {
|
||||
const msg = `Self info file for p2p does not exist! (${paths.selfInfoFile}) P2P functionality will not be loaded`
|
||||
logger.Log(msg, 'redbg')
|
||||
return {}
|
||||
}
|
||||
|
||||
let publicKey: string
|
||||
let privateKey: string
|
||||
|
||||
|
|
@ -483,7 +486,7 @@ function setup(data: SubmoduleData): Submodule {
|
|||
selfInfo: { ...selfInfo, publicKey: publicKey },
|
||||
myPeers: peers.map((peer) => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
const { pw, ...restOfPeer } = peer
|
||||
const { pw, sessionCookie, ...restOfPeer } = peer
|
||||
return restOfPeer
|
||||
}),
|
||||
}
|
||||
|
|
@ -571,7 +574,7 @@ function setup(data: SubmoduleData): Submodule {
|
|||
rawNewQuestionDbs.push(remoteQdb)
|
||||
} else {
|
||||
mergeJobs.push(
|
||||
doALongTask({
|
||||
queueWork({
|
||||
type: 'merge',
|
||||
data: {
|
||||
localQdbIndex: localQdb.index,
|
||||
|
|
@ -630,6 +633,7 @@ function setup(data: SubmoduleData): Submodule {
|
|||
const lastSyncWithPeer = peer.lastSync || 0
|
||||
|
||||
return authAndGetNewData({
|
||||
peers: peers,
|
||||
peer: peer,
|
||||
selfInfo: selfInfo,
|
||||
lastSyncWithPeer: lastSyncWithPeer,
|
||||
|
|
@ -844,21 +848,10 @@ function setup(data: SubmoduleData): Submodule {
|
|||
newQuestions: newQuestionCount,
|
||||
}
|
||||
// Processing result data is successfull
|
||||
const updatedPeersFile = peers.map((x) => {
|
||||
if (isPeerSameAs(peer, x)) {
|
||||
return {
|
||||
...x,
|
||||
lastSync: syncStart,
|
||||
}
|
||||
} else {
|
||||
return x
|
||||
}
|
||||
updatePeersFile(peers, {
|
||||
...peer,
|
||||
lastSync: syncStart,
|
||||
})
|
||||
|
||||
utils.WriteFile(
|
||||
JSON.stringify(updatedPeersFile, null, 2),
|
||||
paths.peersFile
|
||||
)
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------
|
||||
|
|
@ -1014,7 +1007,6 @@ function setup(data: SubmoduleData): Submodule {
|
|||
remotePublicKey,
|
||||
JSON.stringify(newUsers)
|
||||
)
|
||||
// TODO: count sent user count
|
||||
logger.Log(
|
||||
`\tSending new users to "${remoteHost}" (encrypted)`,
|
||||
'green'
|
||||
|
|
@ -1050,7 +1042,6 @@ function setup(data: SubmoduleData): Submodule {
|
|||
? 'all time'
|
||||
: new Date(since).toLocaleString()
|
||||
|
||||
// TODO: count sent data
|
||||
logger.Log(
|
||||
`\tSending new data to ${logger.C(
|
||||
'blue'
|
||||
|
|
|
|||
|
|
@ -48,11 +48,6 @@ import {
|
|||
editDb,
|
||||
RecievedData,
|
||||
} from '../../../utils/actions'
|
||||
import {
|
||||
WorkerResult,
|
||||
// compareQuestionObj,
|
||||
} from '../../../utils/classes'
|
||||
import { doALongTask, msgAllWorker } from '../../../utils/workerPool'
|
||||
import dbtools from '../../../utils/dbtools'
|
||||
import {
|
||||
dataToString,
|
||||
|
|
@ -65,6 +60,8 @@ import {
|
|||
isJsonValidAndLogError,
|
||||
TestUsersSchema,
|
||||
} from '../../../types/typeSchemas'
|
||||
import { msgAllWorker, queueWork } from '../../../worker/workerPool'
|
||||
import { WorkerResult } from '../../../worker/worker'
|
||||
|
||||
interface SavedQuestionData {
|
||||
fname: string
|
||||
|
|
@ -74,13 +71,12 @@ interface SavedQuestionData {
|
|||
date: string | Date
|
||||
}
|
||||
|
||||
// interface SavedQuestion {
|
||||
// Questions: Question[]
|
||||
// subj: string
|
||||
// userid: number
|
||||
// testUrl: string
|
||||
// date: string
|
||||
// }
|
||||
export interface QuestionAddResponse {
|
||||
success: boolean
|
||||
newQuestions: number
|
||||
totalNewQuestions: number
|
||||
result?: string
|
||||
}
|
||||
|
||||
const line = '====================================================' // lol
|
||||
|
||||
|
|
@ -163,8 +159,8 @@ function searchInDbs(
|
|||
// searchIn could be [0], [1], ... to search every db in different thread. Put this into a
|
||||
// forEach(qdbs) to achieve this
|
||||
return new Promise((resolve) => {
|
||||
doALongTask({
|
||||
type: 'work',
|
||||
queueWork({
|
||||
type: 'search',
|
||||
data: {
|
||||
searchIn: searchIn,
|
||||
testUrl: testUrl,
|
||||
|
|
@ -553,7 +549,9 @@ function setup(data: SubmoduleData): Submodule {
|
|||
|
||||
writeIsAddingData(req.body)
|
||||
|
||||
const location = req.body.location.split('/')[2]
|
||||
const location = req.body.location.includes('/')
|
||||
? req.body.location.split('/')[2]
|
||||
: req.body.location
|
||||
|
||||
try {
|
||||
let maxIndex = -1
|
||||
|
|
@ -603,17 +601,27 @@ function setup(data: SubmoduleData): Submodule {
|
|||
|
||||
res.json({
|
||||
success: resultArray.length > 0,
|
||||
newQuestions: resultArray,
|
||||
newQuestions: resultArray, // FIXME: this is useless?
|
||||
totalNewQuestions: totalNewQuestions,
|
||||
})
|
||||
|
||||
if (totalNewQuestions > 0) {
|
||||
resultArray.forEach((result) => {
|
||||
msgAllWorker({
|
||||
// TODO: recognize base64 image
|
||||
type: 'newQuestions',
|
||||
data: result,
|
||||
})
|
||||
if (result.newQuestions.length > 0) {
|
||||
msgAllWorker({
|
||||
type: 'newQuestions',
|
||||
data: result,
|
||||
})
|
||||
if (req.body.fromPeer) return
|
||||
queueWork({
|
||||
type: 'sendQuestionsToPeers',
|
||||
data: {
|
||||
newQuestions: result.newQuestions,
|
||||
subj: result.subjName,
|
||||
location: location, // TODO: location undefined?
|
||||
},
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue