mirror of
https://gitlab.com/MrFry/mrfrys-node-server
synced 2025-04-01 20:24:18 +02:00
added nearly complete p2p implementation
This commit is contained in:
mrfry
parent
11dacdae64
commit
5c22f575dd
25 changed files with 14320 additions and 12563 deletions
23424
package-lock.json
generated
23424
package-lock.json
generated
File diff suppressed because it is too large
Load diff
|
|
@ -9,6 +9,7 @@
|
|||
"ejs": "^3.1.6",
|
||||
"express": "^4.17.3",
|
||||
"express-fileupload": "^1.3.1",
|
||||
"hybrid-crypto-js": "^0.2.4",
|
||||
"socket.io": "^4.4.1",
|
||||
"tesseract.js": "^3.0.3",
|
||||
"ts-node": "^10.7.0",
|
||||
|
|
@ -18,7 +19,7 @@
|
|||
},
|
||||
"scripts": {
|
||||
"start": "node ./dist/server.js",
|
||||
"dev": "npm run build && NS_THREAD_COUNT=2 NS_DEVEL=1 NS_NOUSER=1 NS_LOGLEVEL=1 node --inspect ./dist/server.js",
|
||||
"dev": "npm run build && NS_THREAD_COUNT=2 NS_DEVEL=1 NS_NOUSER=1 node --inspect ./dist/server.js",
|
||||
"build": "tsc && bash -c './scripts/postBuild.sh'",
|
||||
"export": "tsc && bash -c './scripts/postBuild.sh'",
|
||||
"test": "NS_NOLOG=1 NS_THREAD_COUNT=1 jest",
|
||||
|
|
|
|||
1
src/declarations.d.ts
vendored
Normal file
1
src/declarations.d.ts
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
declare module 'hybrid-crypto-js'
|
||||
|
|
@ -19,7 +19,7 @@
|
|||
------------------------------------------------------------------------- */
|
||||
|
||||
import type { Response, NextFunction, RequestHandler } from 'express'
|
||||
import type { Request } from '../types/basicTypes'
|
||||
import type { Request, User } from '../types/basicTypes'
|
||||
import type { Database } from 'better-sqlite3'
|
||||
|
||||
import logger from '../utils/logger'
|
||||
|
|
@ -32,11 +32,13 @@ interface Options {
|
|||
exceptions: Array<string>
|
||||
}
|
||||
|
||||
export const testUser = {
|
||||
export const testUser: User = {
|
||||
id: 19,
|
||||
avaiblePWRequests: 645,
|
||||
pwRequestCount: 19,
|
||||
created: new Date(),
|
||||
created: new Date().getTime(),
|
||||
lastLogin: new Date().getTime(),
|
||||
lastAccess: new Date().getTime(),
|
||||
pw: '5d146f72-e1b8-4440-a6e3-f22f31810316',
|
||||
loginCount: 3,
|
||||
createdBy: 1,
|
||||
|
|
|
|||
|
|
@ -30,7 +30,16 @@ import logger from '../../utils/logger'
|
|||
import utils from '../../utils/utils'
|
||||
import auth from '../../middlewares/auth.middleware'
|
||||
import { SetupData } from '../../server'
|
||||
import { ModuleType, Request, Submodule } from '../../types/basicTypes'
|
||||
import {
|
||||
DataFile,
|
||||
ModuleSpecificData,
|
||||
ModuleType,
|
||||
QuestionDb,
|
||||
Request,
|
||||
Submodule,
|
||||
} from '../../types/basicTypes'
|
||||
import { loadJSON } from '../../utils/actions'
|
||||
import { initWorkerPool } from '../../utils/workerPool'
|
||||
|
||||
// files
|
||||
const rootRedirectToFile = 'data/apiRootRedirectTo'
|
||||
|
|
@ -142,7 +151,23 @@ function GetApp(): ModuleType {
|
|||
|
||||
// -------------------------------------------------------------------------------------------
|
||||
|
||||
const submoduleDatas = setupSubModules(app)
|
||||
const dbsFile = publicDir + 'questionDbs.json'
|
||||
|
||||
// FIXME: is dataFiles only a temp variable? does this cause any problems?
|
||||
const dataFiles: Array<DataFile> = utils.ReadJSON(dbsFile)
|
||||
let questionDbs: Array<QuestionDb> = loadJSON(dataFiles, publicDir)
|
||||
initWorkerPool(() => questionDbs)
|
||||
|
||||
const submoduleDatas = setupSubModules(app, {
|
||||
questionDbs: questionDbs,
|
||||
getQuestionDbs: () => {
|
||||
return questionDbs
|
||||
},
|
||||
setQuestionDbs: (newQdbs: QuestionDb[]) => {
|
||||
questionDbs = newQdbs
|
||||
},
|
||||
dbsFile: dbsFile,
|
||||
})
|
||||
|
||||
// -------------------------------------------------------------------------------------------
|
||||
|
||||
|
|
@ -183,7 +208,7 @@ function GetApp(): ModuleType {
|
|||
|
||||
function setupSubModules(
|
||||
parentApp: express.Application,
|
||||
moduleSpecificData?: any
|
||||
moduleSpecificData: ModuleSpecificData
|
||||
): Submodule[] {
|
||||
const submoduleDir = './submodules/'
|
||||
const absolutePath = __dirname + '/' + submoduleDir
|
||||
|
|
|
|||
955
src/modules/api/submodules/p2p.ts
Normal file
955
src/modules/api/submodules/p2p.ts
Normal file
|
|
@ -0,0 +1,955 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
|
||||
Question Server
|
||||
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
------------------------------------------------------------------------- */
|
||||
|
||||
import { Response } from 'express'
|
||||
import * as child_process from 'child_process'
|
||||
import http from 'http'
|
||||
|
||||
import logger from '../../../utils/logger'
|
||||
import {
|
||||
Request,
|
||||
SubmoduleData,
|
||||
Submodule,
|
||||
PeerInfo,
|
||||
Subject,
|
||||
QuestionDb,
|
||||
User,
|
||||
} from '../../../types/basicTypes'
|
||||
import utils from '../../../utils/utils'
|
||||
import { backupData /*writeData*/ } from '../../../utils/actions'
|
||||
import { WorkerResult } from '../../../utils/classes'
|
||||
import dbtools from '../../../utils/dbtools'
|
||||
import {
|
||||
createKeyPair,
|
||||
decrypt,
|
||||
encrypt,
|
||||
isKeypairValid,
|
||||
} from '../../../utils/encryption'
|
||||
import { doALongTask, msgAllWorker } from '../../../utils/workerPool'
|
||||
import {
|
||||
countOfQdb,
|
||||
countOfQdbs,
|
||||
createQuestion,
|
||||
getAvailableQdbIndexes,
|
||||
removeCacheFromQuestion,
|
||||
} from '../../../utils/qdbUtils'
|
||||
|
||||
// TODO: remove FINALIZE-s and TOTEST-s
|
||||
// TODO: script to remove from date from certain host (questions / users)
|
||||
|
||||
interface MergeResult {
|
||||
newData: Subject[]
|
||||
newSubjects: Subject[]
|
||||
localQdbIndex: number
|
||||
e: Error
|
||||
}
|
||||
|
||||
interface RemotePeerInfo {
|
||||
selfInfo: PeerInfo
|
||||
myPeers: PeerInfo[]
|
||||
revision?: string
|
||||
qdbInfo?: {
|
||||
dbName: string
|
||||
subjs: {
|
||||
name: string
|
||||
count: number
|
||||
}[]
|
||||
}[]
|
||||
}
|
||||
|
||||
interface RequestResult<T> {
|
||||
data?: T
|
||||
error?: Error
|
||||
options?: http.RequestOptions
|
||||
}
|
||||
|
||||
interface SyncDataRes {
|
||||
questionDbs?: QuestionDb[]
|
||||
remoteInfo?: RemotePeerInfo
|
||||
encryptedUsers?: string
|
||||
count: {
|
||||
qdbs: number
|
||||
subjects: number
|
||||
questions: number
|
||||
}
|
||||
}
|
||||
|
||||
function get<T>(options: http.RequestOptions): Promise<RequestResult<T>> {
|
||||
return new Promise((resolve) => {
|
||||
const req = http.get(options, function (res) {
|
||||
const bodyChunks: Uint8Array[] = []
|
||||
res.on('data', (chunk) => {
|
||||
bodyChunks.push(chunk)
|
||||
}).on('end', () => {
|
||||
const body = Buffer.concat(bodyChunks).toString()
|
||||
try {
|
||||
resolve({ data: JSON.parse(body) })
|
||||
} catch (e) {
|
||||
resolve({ error: e, options: options })
|
||||
}
|
||||
})
|
||||
})
|
||||
req.on('error', function (e) {
|
||||
resolve({ error: e, options: options })
|
||||
// reject(e)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function peerToString(peer: PeerInfo) {
|
||||
return `${peer.host}:${peer.port}`
|
||||
}
|
||||
|
||||
function isPeerSameAs(peer1: PeerInfo, peer2: PeerInfo) {
|
||||
return peer1.host === peer2.host && peer1.port === peer2.port
|
||||
}
|
||||
|
||||
export function getNewDataSince(subjects: Subject[], date: number): Subject[] {
|
||||
return subjects
|
||||
.map((subject) => {
|
||||
return {
|
||||
...subject,
|
||||
Questions: subject.Questions.filter((question) => {
|
||||
return (question.data.date || 0) > date
|
||||
}).map((question) => removeCacheFromQuestion(question)),
|
||||
}
|
||||
})
|
||||
.filter((subject) => subject.Questions.length !== 0)
|
||||
}
|
||||
|
||||
export function mergeSubjects(
|
||||
subjectsToMergeTo: Subject[],
|
||||
subjectsToMerge: Subject[],
|
||||
newSubjects: Subject[]
|
||||
): Subject[] {
|
||||
return [
|
||||
...subjectsToMergeTo.map((subj) => {
|
||||
const newSubjs = subjectsToMerge.filter(
|
||||
(subjRes) => subjRes.Name === subj.Name
|
||||
)
|
||||
|
||||
if (newSubjs) {
|
||||
const newQuestions = newSubjs.flatMap((subj) => {
|
||||
return subj.Questions
|
||||
})
|
||||
|
||||
return {
|
||||
...subj,
|
||||
Questions: [...subj.Questions, ...newQuestions],
|
||||
}
|
||||
} else {
|
||||
return subj
|
||||
}
|
||||
}),
|
||||
...newSubjects,
|
||||
]
|
||||
}
|
||||
|
||||
export function mergeQdbs(
|
||||
qdbToMergeTo: QuestionDb[],
|
||||
mergeResults: MergeResult[]
|
||||
): { mergedQuestionDbs: QuestionDb[]; changedQdbIndexes: number[] } {
|
||||
const changedQdbIndexes: number[] = []
|
||||
const mergedQuestionDbs = qdbToMergeTo.map((qdb) => {
|
||||
const qdbMergeResult = mergeResults.find(
|
||||
(mergeRes) => mergeRes.localQdbIndex === qdb.index
|
||||
)
|
||||
if (qdbMergeResult) {
|
||||
const mergedQdb = {
|
||||
...qdb,
|
||||
data: mergeSubjects(
|
||||
qdb.data,
|
||||
qdbMergeResult.newData,
|
||||
qdbMergeResult.newSubjects
|
||||
),
|
||||
}
|
||||
changedQdbIndexes.push(qdb.index)
|
||||
return mergedQdb
|
||||
} else {
|
||||
// unchanged
|
||||
return qdb
|
||||
}
|
||||
})
|
||||
return {
|
||||
mergedQuestionDbs: mergedQuestionDbs,
|
||||
changedQdbIndexes: changedQdbIndexes,
|
||||
}
|
||||
}
|
||||
|
||||
async function sendNewDataToWorkers(
|
||||
mergeResults: MergeResult[],
|
||||
newQuestionDbs: QuestionDb[]
|
||||
) {
|
||||
// FIXME: this might be slow, maybe make a new type of message for workers?
|
||||
const updatePromises: Promise<WorkerResult[]>[] = []
|
||||
let newQuestionCount = 0
|
||||
let newSubjectCount = 0
|
||||
let newQuestionDbCount = 0
|
||||
|
||||
mergeResults.forEach((mergeRes) => {
|
||||
if (mergeRes.e) {
|
||||
logger.Log(`There was an error processing the merge!`, 'redbg')
|
||||
console.error(mergeRes.e)
|
||||
return
|
||||
}
|
||||
|
||||
mergeRes.newData.forEach((subjectWithNewData) => {
|
||||
newQuestionCount += subjectWithNewData.Questions.length
|
||||
updatePromises.push(
|
||||
msgAllWorker({
|
||||
type: 'newQuestions',
|
||||
data: {
|
||||
subjName: subjectWithNewData.Name,
|
||||
qdbIndex: mergeRes.localQdbIndex,
|
||||
newQuestions: subjectWithNewData.Questions,
|
||||
},
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
newSubjectCount += mergeRes.newSubjects.length
|
||||
mergeRes.newSubjects.forEach((newSubject) => {
|
||||
newQuestionCount += newSubject.Questions.length
|
||||
updatePromises.push(
|
||||
msgAllWorker({
|
||||
type: 'newQuestions',
|
||||
data: {
|
||||
subjName: newSubject.Name,
|
||||
qdbIndex: mergeRes.localQdbIndex,
|
||||
newQuestions: newSubject.Questions,
|
||||
},
|
||||
})
|
||||
)
|
||||
})
|
||||
})
|
||||
newQuestionDbCount += newQuestionDbs.length
|
||||
newQuestionDbs.forEach((newQdb) => {
|
||||
const { subjCount: sc, questionCount: qc } = countOfQdb(newQdb)
|
||||
newSubjectCount += sc
|
||||
newQuestionCount += qc
|
||||
msgAllWorker({
|
||||
data: newQdb,
|
||||
type: 'newdb',
|
||||
})
|
||||
})
|
||||
|
||||
await Promise.all(updatePromises)
|
||||
|
||||
return {
|
||||
newQuestionDbCount: newQuestionDbCount,
|
||||
newSubjectCount: newSubjectCount,
|
||||
newQuestionCount: newQuestionCount,
|
||||
}
|
||||
}
|
||||
|
||||
function writeNewData(
|
||||
newQuestionDbs: QuestionDb[],
|
||||
changedQuestionDbs: QuestionDb[]
|
||||
) {
|
||||
const qdbsToWrite = [...newQuestionDbs, ...changedQuestionDbs]
|
||||
qdbsToWrite.forEach((qdb) => {
|
||||
try {
|
||||
// FINALIZE: write to file
|
||||
// writeData(qdb.data, qdb.path)
|
||||
} catch (e) {
|
||||
logger.Log(`Error writing ${qdb.name} qdb to file!`, 'redbg')
|
||||
console.error(e)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function updateLastSync(selfInfo: PeerInfo, newDate: number) {
|
||||
utils.WriteFile(
|
||||
JSON.stringify({ ...selfInfo, lastSync: newDate }, null, 2),
|
||||
selfInfoFile
|
||||
)
|
||||
}
|
||||
|
||||
function setupQuestionsForMerge(qdb: QuestionDb, peer: PeerInfo) {
|
||||
return {
|
||||
...qdb,
|
||||
data: qdb.data.map((subj) => {
|
||||
return {
|
||||
...subj,
|
||||
Questions: subj.Questions.map((q) => {
|
||||
const initializedQuestion = q.cache ? q : createQuestion(q)
|
||||
initializedQuestion.data.source = peerToString(peer)
|
||||
return initializedQuestion
|
||||
}),
|
||||
}
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
// files
|
||||
const peersPath = 'data/p2p/'
|
||||
const peersFile = peersPath + '/peers.json'
|
||||
// writes it)
|
||||
const selfInfoFile = peersPath + '/selfInfo.json'
|
||||
const thirdPartyPeersFile = peersPath + '/thirdPartyPeers.json'
|
||||
const keyFile = peersPath + '/key' // key.pub key.priv
|
||||
|
||||
function setup(data: SubmoduleData): Submodule {
|
||||
const {
|
||||
app,
|
||||
userDB,
|
||||
publicdirs,
|
||||
moduleSpecificData: { questionDbs, setQuestionDbs, getQuestionDbs },
|
||||
// publicdirs,
|
||||
} = data
|
||||
|
||||
const publicDir = publicdirs[0]
|
||||
|
||||
// ---------------------------------------------------------------------------------------
|
||||
// SETUP
|
||||
// ---------------------------------------------------------------------------------------
|
||||
|
||||
// const publicDir = publicdirs[0]
|
||||
|
||||
if (!utils.FileExists(peersFile)) {
|
||||
logger.Log(
|
||||
`Warning: peers file was missing, so it was created`,
|
||||
'yellowbg'
|
||||
)
|
||||
utils.CreatePath(peersPath)
|
||||
utils.WriteFile('[]', peersFile)
|
||||
}
|
||||
|
||||
if (!utils.FileExists(selfInfoFile)) {
|
||||
logger.Log(
|
||||
'Self info file for p2p does not exist! P2P functionality will not be loaded',
|
||||
'redbg'
|
||||
)
|
||||
logger.Log(
|
||||
`File should be at: ${selfInfoFile} with the interface 'PeerInfo'`
|
||||
)
|
||||
throw new Error('p2p error')
|
||||
}
|
||||
|
||||
let publicKey: string
|
||||
let privateKey: string
|
||||
|
||||
if (
|
||||
!utils.FileExists(keyFile + '.priv') ||
|
||||
!utils.FileExists(keyFile + '.pub')
|
||||
) {
|
||||
createKeyPair().then(({ publicKey: pubk, privateKey: privk }) => {
|
||||
// at first start there won't be a keypair available until this finishes
|
||||
utils.WriteFile(pubk, keyFile + '.pub')
|
||||
utils.WriteFile(privk, keyFile + '.priv')
|
||||
|
||||
publicKey = pubk
|
||||
privateKey = privk
|
||||
})
|
||||
logger.Log(
|
||||
'There were no public / private keys for p2p functionality, created new ones',
|
||||
'yellowbg'
|
||||
)
|
||||
} else {
|
||||
publicKey = utils.ReadFile(keyFile + '.pub')
|
||||
privateKey = utils.ReadFile(keyFile + '.priv')
|
||||
// checking only here, because if it got generated in the other branch then it must be good
|
||||
if (!isKeypairValid(publicKey, privateKey)) {
|
||||
logger.Log('Loaded keypair is not valid!', 'redbg')
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: validate peers
|
||||
let peers: PeerInfo[] = utils.ReadJSON(peersFile)
|
||||
let selfInfo: PeerInfo = utils.ReadJSON(selfInfoFile)
|
||||
// self info file is not required to have the publicKey, as it is always added on init
|
||||
selfInfo.publicKey = publicKey
|
||||
|
||||
const filesToWatch = [
|
||||
{
|
||||
fname: peersFile,
|
||||
logMsg: 'Peers file updated',
|
||||
action: () => {
|
||||
peers = utils.ReadJSON(peersFile)
|
||||
},
|
||||
},
|
||||
{
|
||||
fname: selfInfoFile,
|
||||
logMsg: 'P2P self info file changed',
|
||||
action: () => {
|
||||
selfInfo = utils.ReadJSON(selfInfoFile)
|
||||
selfInfo.publicKey = publicKey
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
filesToWatch.forEach((ftw) => {
|
||||
if (utils.FileExists(ftw.fname)) {
|
||||
utils.WatchFile(ftw.fname, () => {
|
||||
logger.Log(ftw.logMsg)
|
||||
ftw.action()
|
||||
})
|
||||
ftw.action()
|
||||
} else {
|
||||
logger.Log(`File ${ftw.fname} does not exists to watch!`, 'redbg')
|
||||
}
|
||||
})
|
||||
|
||||
if (peers.length === 0) {
|
||||
logger.Log(
|
||||
`Warning: peers file is empty. You probably want to fill it`,
|
||||
'yellowbg'
|
||||
)
|
||||
} else {
|
||||
logger.Log('Loaded peers: ' + peers.length)
|
||||
peers.forEach((peer, i) => {
|
||||
logger.Log(`\t${i}\t"${peer.name}": ${peerToString(peer)}`)
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------
|
||||
// FUNCTIONS
|
||||
// ---------------------------------------------------------------------------------------
|
||||
|
||||
function getSelfInfo(includeQdbInfo?: boolean) {
|
||||
const result: RemotePeerInfo = {
|
||||
selfInfo: selfInfo,
|
||||
myPeers: peers,
|
||||
}
|
||||
|
||||
try {
|
||||
// FIXME: dont log if fails
|
||||
result.revision = child_process
|
||||
.execSync('git rev-parse HEAD', { cwd: __dirname })
|
||||
.toString()
|
||||
.trim()
|
||||
} catch (e) {
|
||||
result.revision = 'Failed to get revision'
|
||||
}
|
||||
|
||||
if (includeQdbInfo) {
|
||||
result.qdbInfo = getQuestionDbs().map((qdb) => {
|
||||
return {
|
||||
dbName: qdb.name,
|
||||
subjs: qdb.data.map((subj) => {
|
||||
return {
|
||||
name: subj.Name,
|
||||
count: subj.Questions.length,
|
||||
}
|
||||
}),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
function getNewUsersSince(since?: number) {
|
||||
const users = dbtools.runStatement(
|
||||
userDB,
|
||||
`SELECT *
|
||||
FROM users
|
||||
WHERE created >= ${since};`
|
||||
)
|
||||
return users
|
||||
}
|
||||
|
||||
function updateQdbForLocalUse(qdb: QuestionDb[]) {
|
||||
const availableIndexes = getAvailableQdbIndexes(
|
||||
getQuestionDbs(),
|
||||
qdb.length
|
||||
)
|
||||
return qdb.map((qdb, i) => {
|
||||
return {
|
||||
...qdb,
|
||||
index: availableIndexes[i],
|
||||
path: `${publicDir}questionDbs/${qdb.name}.json'`,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async function getMergeResults(remoteQuestionDbs: QuestionDb[]) {
|
||||
const mergeJobs: Promise<any>[] = []
|
||||
const rawNewQuestionDbs: QuestionDb[] = []
|
||||
remoteQuestionDbs.forEach((remoteQdb) => {
|
||||
// TODO: warn on qdb differences like shouldSave
|
||||
const localQdb = getQuestionDbs().find(
|
||||
(lqdb) => lqdb.name === remoteQdb.name
|
||||
)
|
||||
|
||||
if (!localQdb) {
|
||||
rawNewQuestionDbs.push(remoteQdb)
|
||||
} else {
|
||||
mergeJobs.push(
|
||||
doALongTask({
|
||||
type: 'merge',
|
||||
data: {
|
||||
localQdbIndex: localQdb.index,
|
||||
remoteQdb: remoteQdb,
|
||||
},
|
||||
})
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
const mergeResults: MergeResult[] = await Promise.all(mergeJobs)
|
||||
|
||||
return {
|
||||
mergeResults: mergeResults,
|
||||
rawNewQuestionDbs: rawNewQuestionDbs,
|
||||
}
|
||||
}
|
||||
|
||||
async function syncData() {
|
||||
// TOTEST: try with 0 date to merge full dbs
|
||||
if (peers.length === 0) {
|
||||
logger.Log(
|
||||
`There are no peers specified in ${peersFile}, aborting sync`,
|
||||
'yellowbg'
|
||||
)
|
||||
return {
|
||||
msg: 'No peers specified, aborting',
|
||||
}
|
||||
}
|
||||
// FIXME: this might be blocking the main thread, but not sure how much
|
||||
logger.Log(
|
||||
`\tStarting data sync, getting new data from ${logger.C('green')}${
|
||||
peers.length
|
||||
}${logger.C()} peers`
|
||||
)
|
||||
|
||||
const lastSync = new Date('2023-01-01').getTime() // FINALIZE date: this is only for testing // selfInfo.lastSync
|
||||
logger.Log(
|
||||
`\tLast sync date: ${logger.C('blue')}${new Date(
|
||||
lastSync
|
||||
).toLocaleString()}${logger.C()}`
|
||||
)
|
||||
const syncStart = new Date().getTime()
|
||||
const requests = peers.map((peer) => {
|
||||
const lastSyncWithPeer = new Date('2023-01-01').getTime() // FINALIZE same as above // peer.lastSync || 0
|
||||
|
||||
logger.Log(
|
||||
`\tLast sync with ${logger.C('blue')}${peerToString(
|
||||
peer
|
||||
)}${logger.C()}: ${logger.C('blue')}${new Date(
|
||||
lastSyncWithPeer
|
||||
).toLocaleString()}${logger.C()}`
|
||||
)
|
||||
return new Promise<RequestResult<SyncDataRes & { peer: PeerInfo }>>(
|
||||
(resolve) => {
|
||||
get<SyncDataRes>({
|
||||
host: peer.host,
|
||||
port: peer.port,
|
||||
path: `/getnewdatasince?host=${selfInfo.host}${
|
||||
lastSync ? `&since=${lastSyncWithPeer}` : ''
|
||||
}`,
|
||||
}).then((res) => {
|
||||
resolve({ ...res, data: { ...res.data, peer: peer } })
|
||||
})
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
const allResults = await Promise.all(requests)
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------
|
||||
// filtering, transforming, and counting data
|
||||
// -------------------------------------------------------------------------------------------------------
|
||||
allResults.forEach((res) => {
|
||||
if (res.error) {
|
||||
logger.Log(
|
||||
`\tError syncing with ${peerToString(res.data.peer)}: ${
|
||||
res.error.message
|
||||
}`,
|
||||
'red'
|
||||
)
|
||||
}
|
||||
})
|
||||
const resultDataWithoutErrors = allResults
|
||||
.filter((res) => !res.error)
|
||||
.map((res) => res.data)
|
||||
|
||||
if (resultDataWithoutErrors.length === 0) {
|
||||
logger.Log(
|
||||
`No peers returned data without error, aborting sync`,
|
||||
'redbg'
|
||||
)
|
||||
return {
|
||||
msg: 'No peers returned data without error, aborting sync',
|
||||
}
|
||||
}
|
||||
|
||||
const resultDataWithoutEmptyDbs = resultDataWithoutErrors.filter(
|
||||
(res) => {
|
||||
const qdbCount = res.questionDbs.length
|
||||
const { subjCount, questionCount } = countOfQdbs(
|
||||
res.questionDbs
|
||||
)
|
||||
|
||||
logger.Log(
|
||||
`\t"${logger.C('blue')}${peerToString(
|
||||
res.peer
|
||||
)}${logger.C()}" sent "${logger.C(
|
||||
'green'
|
||||
)}${qdbCount}${logger.C()}" question DB-s with "${logger.C(
|
||||
'green'
|
||||
)}${subjCount.toLocaleString()}${logger.C()}" subjects, and "${logger.C(
|
||||
'green'
|
||||
)}${questionCount.toLocaleString()}${logger.C()}" questions`
|
||||
)
|
||||
|
||||
return questionCount > 0
|
||||
}
|
||||
)
|
||||
|
||||
// TOTEST: even on new subjet and new qdb add! TEST
|
||||
// add new quesstions to db (QuestionData.source = true)
|
||||
const resultData = resultDataWithoutEmptyDbs.map((res) => {
|
||||
return {
|
||||
...res,
|
||||
questionDbs: res.questionDbs.map((qdb) => {
|
||||
return setupQuestionsForMerge(qdb, res.peer)
|
||||
}),
|
||||
}
|
||||
})
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------
|
||||
// third party peers handling
|
||||
// -------------------------------------------------------------------------------------------------------
|
||||
const peersHosts = [...peers.map((peer) => peer.host), selfInfo.host]
|
||||
const thirdPartyPeers = resultData
|
||||
.map((res) => res.remoteInfo)
|
||||
.flatMap((x) => {
|
||||
return x.myPeers.filter(
|
||||
(recievedPeer) => !peersHosts.includes(recievedPeer.host)
|
||||
)
|
||||
})
|
||||
if (thirdPartyPeers.length > 0) {
|
||||
logger.Log(
|
||||
`\tPeers reported ${logger.C('green')}${
|
||||
thirdPartyPeers.length
|
||||
}${logger.C()} third party peer(s) not connected to this server.`
|
||||
)
|
||||
utils.WriteFile(
|
||||
JSON.stringify(thirdPartyPeers, null, 2),
|
||||
thirdPartyPeersFile
|
||||
)
|
||||
logger.Log(
|
||||
`\tSee ${logger.C(
|
||||
'blue'
|
||||
)}${thirdPartyPeersFile}${logger.C()} for details`
|
||||
)
|
||||
}
|
||||
// -------------------------------------------------------------------------------------------------------
|
||||
// new users handlin TOTEST: test
|
||||
// -------------------------------------------------------------------------------------------------------
|
||||
let newUsers = 0
|
||||
const oldUserCount = dbtools.SelectAll(userDB, 'users').length
|
||||
try {
|
||||
resultData.forEach((res) => {
|
||||
if (res.encryptedUsers) {
|
||||
const decryptedUsers: User[] = JSON.parse(
|
||||
decrypt(privateKey, res.encryptedUsers)
|
||||
)
|
||||
let newUserCount = 0
|
||||
decryptedUsers.forEach((remoteUser) => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
const { id, ...remoteUserWithoutId } = remoteUser
|
||||
const localUser = dbtools.Select(userDB, 'users', {
|
||||
pw: remoteUser.pw,
|
||||
})
|
||||
if (!localUser) {
|
||||
// FIXME: users will not have consistend id across servers. This may be
|
||||
// harmless, will see
|
||||
dbtools.Insert(userDB, 'users', {
|
||||
...(remoteUserWithoutId as Omit<User, 'id'>),
|
||||
sourceHost: peerToString(res.peer),
|
||||
})
|
||||
newUserCount += 1
|
||||
}
|
||||
})
|
||||
if (newUserCount > 0) {
|
||||
newUsers += newUserCount
|
||||
logger.Log(
|
||||
`\tAdded ${logger.C(
|
||||
'green'
|
||||
)}${newUserCount}${logger.C()} users from "${logger.C(
|
||||
'blue'
|
||||
)}${peerToString(res.peer)}${logger.C()}"`
|
||||
)
|
||||
}
|
||||
}
|
||||
})
|
||||
} catch (e) {
|
||||
logger.Log(
|
||||
'\tError while trying to sync users: ' + e.message,
|
||||
'redbg'
|
||||
)
|
||||
console.error(e)
|
||||
}
|
||||
const newUserCount = dbtools.SelectAll(userDB, 'users').length
|
||||
|
||||
const hasNewData = resultData.length > 0
|
||||
if (!hasNewData) {
|
||||
logger.Log(
|
||||
`No peers returned any new questions. Sync successfully finished!`,
|
||||
'green'
|
||||
)
|
||||
updateLastSync(selfInfo, syncStart)
|
||||
return {
|
||||
msg: 'No peers returned any new questions',
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------
|
||||
// backup
|
||||
// -------------------------------------------------------------------------------------------------------
|
||||
|
||||
const { subjCount: oldSubjCount, questionCount: oldQuestionCount } =
|
||||
countOfQdbs(getQuestionDbs())
|
||||
const oldQuestionDbCount = getQuestionDbs().length
|
||||
// TOTEST: test if backup wrks
|
||||
logger.Log('\tBacking up old data ...')
|
||||
backupData(getQuestionDbs())
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------
|
||||
// adding questions to db
|
||||
// -------------------------------------------------------------------------------------------------------
|
||||
|
||||
let totalNewQuestions = 0
|
||||
let totalNewSubjects = 0
|
||||
let totalNewQdbs = 0
|
||||
for (let i = 0; i < resultData.length; i++) {
|
||||
const { questionDbs: remoteQuestionDbs, peer } = resultData[i]
|
||||
// FIXME: if remoteQuestionDbs contain multiple dbs with the same name, then the merging
|
||||
// process could get wonky. Ideally it should not contain, but we will see
|
||||
|
||||
const { rawNewQuestionDbs, mergeResults } = await getMergeResults(
|
||||
remoteQuestionDbs
|
||||
)
|
||||
|
||||
const newQuestionDbs = updateQdbForLocalUse(rawNewQuestionDbs)
|
||||
|
||||
const { mergedQuestionDbs, changedQdbIndexes } = mergeQdbs(
|
||||
getQuestionDbs(),
|
||||
mergeResults
|
||||
)
|
||||
// TOTEST: test muliple new question dbs from multiple sources
|
||||
// setting new index & path
|
||||
writeNewData(
|
||||
newQuestionDbs,
|
||||
getQuestionDbs().filter((qdb) => {
|
||||
return changedQdbIndexes.includes(qdb.index)
|
||||
})
|
||||
)
|
||||
|
||||
setQuestionDbs([...mergedQuestionDbs, ...newQuestionDbs])
|
||||
|
||||
const { newQuestionDbCount, newSubjectCount, newQuestionCount } =
|
||||
await sendNewDataToWorkers(mergeResults, newQuestionDbs)
|
||||
|
||||
if (newQuestionCount > 0) {
|
||||
logger.Log(
|
||||
`\tAdded ${logger.C(
|
||||
'green'
|
||||
)}${newQuestionDbCount.toLocaleString()}${logger.C()} new question DB-s, ${logger.C(
|
||||
'green'
|
||||
)}${newSubjectCount.toLocaleString()}${logger.C()} new subjects and ${logger.C(
|
||||
'green'
|
||||
)}${newQuestionCount.toLocaleString()}${logger.C()} new questions from "${logger.C(
|
||||
'blue'
|
||||
)}${peerToString(peer)}${logger.C()}"`
|
||||
)
|
||||
}
|
||||
|
||||
// Processing result data is successfull
|
||||
const newPeers = peers.map((x) => {
|
||||
if (isPeerSameAs(peer, x)) {
|
||||
return {
|
||||
...x,
|
||||
lastSync: syncStart,
|
||||
}
|
||||
} else {
|
||||
return x
|
||||
}
|
||||
})
|
||||
|
||||
utils.WriteFile(JSON.stringify(newPeers, null, 2), peersFile)
|
||||
|
||||
totalNewQdbs += newQuestionDbCount
|
||||
totalNewSubjects += newSubjectCount
|
||||
totalNewQuestions += newQuestionCount
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------
|
||||
updateLastSync(selfInfo, syncStart)
|
||||
|
||||
const newQdb = getQuestionDbs()
|
||||
const { subjCount: newSubjCount, questionCount: newQuestionCount } =
|
||||
countOfQdbs(newQdb)
|
||||
const newQuestionDbCount = newQdb.length
|
||||
|
||||
logger.logTable([
|
||||
['\t', 'Users', 'QDBs', 'Subjs', 'Questions'],
|
||||
[
|
||||
'Old\t',
|
||||
oldUserCount,
|
||||
oldQuestionDbCount,
|
||||
oldSubjCount,
|
||||
oldQuestionCount,
|
||||
],
|
||||
[
|
||||
'Added',
|
||||
newUsers,
|
||||
totalNewQdbs,
|
||||
totalNewSubjects,
|
||||
totalNewQuestions,
|
||||
],
|
||||
[
|
||||
'Final',
|
||||
newUserCount,
|
||||
newQuestionDbCount,
|
||||
newSubjCount,
|
||||
newQuestionCount,
|
||||
],
|
||||
])
|
||||
|
||||
logger.Log(
|
||||
`Question DB-s written! Sync successfully finished!`,
|
||||
'green'
|
||||
)
|
||||
|
||||
return {
|
||||
old: {
|
||||
oldUserCount: oldUserCount,
|
||||
oldQuestionDbCount: oldQuestionDbCount,
|
||||
oldSubjCount: oldSubjCount,
|
||||
oldQuestionCount: oldQuestionCount,
|
||||
},
|
||||
added: {
|
||||
totalNewQdbs: totalNewQdbs,
|
||||
totalNewSubjects: totalNewSubjects,
|
||||
totalNewQuestions: totalNewQuestions,
|
||||
},
|
||||
final: {
|
||||
newUserCount: newUserCount,
|
||||
newQuestionDbCount: newQuestionDbCount,
|
||||
newSubjCount: newSubjCount,
|
||||
newQuestionCount: newQuestionCount,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------
|
||||
// APP SETUP
|
||||
// ---------------------------------------------------------------------------------------
|
||||
app.get('/p2pinfo', (req: Request, res: Response<RemotePeerInfo>) => {
|
||||
logger.LogReq(req)
|
||||
res.json(getSelfInfo(true))
|
||||
})
|
||||
|
||||
app.get('/getnewdatasince', (req: Request, res: Response<SyncDataRes>) => {
|
||||
// TODO: hash question db to see if different?
|
||||
logger.LogReq(req)
|
||||
const since = +req.query.since
|
||||
const remoteHost = req.query.host
|
||||
|
||||
const questionDbsWithNewQuestions = Number.isNaN(since)
|
||||
? questionDbs
|
||||
: questionDbs
|
||||
.map((qdb) => {
|
||||
return {
|
||||
...qdb,
|
||||
data: getNewDataSince(qdb.data, since),
|
||||
}
|
||||
})
|
||||
.filter((qdb) => {
|
||||
const { questionCount: questionCount } = countOfQdb(qdb)
|
||||
return questionCount > 0
|
||||
})
|
||||
|
||||
const { subjCount: subjects, questionCount: questions } = countOfQdbs(
|
||||
questionDbsWithNewQuestions
|
||||
)
|
||||
|
||||
const result: SyncDataRes = {
|
||||
questionDbs: questionDbsWithNewQuestions,
|
||||
count: {
|
||||
qdbs: questionDbsWithNewQuestions.length,
|
||||
subjects: subjects,
|
||||
questions: questions,
|
||||
},
|
||||
remoteInfo: getSelfInfo(),
|
||||
}
|
||||
|
||||
if (remoteHost) {
|
||||
const remoteHostInfo = peers.find((peer) => {
|
||||
return peer.host === remoteHost
|
||||
})
|
||||
const remotePublicKey = remoteHostInfo?.publicKey
|
||||
if (remotePublicKey) {
|
||||
// FIXME: sign data?
|
||||
const newUsers = getNewUsersSince(since)
|
||||
result.encryptedUsers = encrypt(
|
||||
remotePublicKey,
|
||||
JSON.stringify(newUsers)
|
||||
)
|
||||
} else if (remoteHostInfo) {
|
||||
logger.Log(
|
||||
`Warning: ${remoteHostInfo.host}:${remoteHostInfo.port} has no publick key saved!`,
|
||||
'yellowbg'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
res.json(result)
|
||||
})
|
||||
|
||||
app.get('/syncp2pdata', (req: Request, res: Response) => {
|
||||
logger.LogReq(req)
|
||||
// FINALIZE: uncomment
|
||||
// const user = req.session.user
|
||||
// if (user.id !== 1) {
|
||||
// res.json({
|
||||
// status: 'error',
|
||||
// msg: 'only user 1 can call this EP',
|
||||
// })
|
||||
// return
|
||||
// }
|
||||
|
||||
syncData()
|
||||
.then((syncResult) => {
|
||||
res.json({
|
||||
msg: 'sync successfull',
|
||||
...syncResult,
|
||||
})
|
||||
})
|
||||
.catch((e) => {
|
||||
console.error(e)
|
||||
res.json({
|
||||
error: e,
|
||||
msg: e.message,
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
logger.Log('P2P functionality set up. Peers: ' + peers.length, 'blue')
|
||||
|
||||
return {}
|
||||
}
|
||||
|
||||
export default {
|
||||
setup: setup,
|
||||
}
|
||||
|
|
@ -44,24 +44,21 @@ import {
|
|||
Result,
|
||||
backupData,
|
||||
shouldSearchDataFile,
|
||||
loadJSON,
|
||||
writeData,
|
||||
editDb,
|
||||
RecievedData,
|
||||
} from '../../../utils/actions'
|
||||
import {
|
||||
dataToString,
|
||||
getSubjNameWithoutYear,
|
||||
WorkerResult,
|
||||
SearchResultQuestion,
|
||||
// compareQuestionObj,
|
||||
} from '../../../utils/classes'
|
||||
import {
|
||||
doALongTask,
|
||||
msgAllWorker,
|
||||
initWorkerPool,
|
||||
} from '../../../utils/workerPool'
|
||||
import { doALongTask, msgAllWorker } from '../../../utils/workerPool'
|
||||
import dbtools from '../../../utils/dbtools'
|
||||
import {
|
||||
dataToString,
|
||||
getSubjNameWithoutYear,
|
||||
SearchResultQuestion,
|
||||
} from '../../../utils/qdbUtils'
|
||||
|
||||
interface SavedQuestionData {
|
||||
fname: string
|
||||
|
|
@ -469,11 +466,15 @@ function getNewQdb(
|
|||
}
|
||||
|
||||
function setup(data: SubmoduleData): Submodule {
|
||||
const { app, userDB, /* url */ publicdirs /* moduleSpecificData */ } = data
|
||||
const {
|
||||
app,
|
||||
userDB,
|
||||
/* url */ publicdirs,
|
||||
moduleSpecificData: { questionDbs: questionDbs, dbsFile: dbsFile },
|
||||
} = data
|
||||
|
||||
const publicDir = publicdirs[0]
|
||||
const motdFile = publicDir + 'motd'
|
||||
const dbsFile = publicDir + 'questionDbs.json'
|
||||
const savedQuestionsDir = publicDir + 'savedQuestions'
|
||||
|
||||
let version = LoadVersion()
|
||||
|
|
@ -481,10 +482,6 @@ function setup(data: SubmoduleData): Submodule {
|
|||
let motd = LoadMOTD(motdFile)
|
||||
let testUsers: number[] = LoadTestUsers()
|
||||
|
||||
const dataFiles: Array<DataFile> = utils.ReadJSON(dbsFile)
|
||||
const questionDbs: Array<QuestionDb> = loadJSON(dataFiles, publicDir)
|
||||
initWorkerPool(() => questionDbs)
|
||||
|
||||
const filesToWatch = [
|
||||
{
|
||||
fname: motdFile,
|
||||
|
|
|
|||
|
|
@ -320,7 +320,7 @@ function setup(data: SubmoduleData): Submodule {
|
|||
}
|
||||
)
|
||||
|
||||
function getDayDiff(dateString: string | Date) {
|
||||
function getDayDiff(dateString: string | Date | number) {
|
||||
const msdiff = new Date().getTime() - new Date(dateString).getTime()
|
||||
return Math.floor(msdiff / (1000 * 3600 * 24))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ console.log('Current working directory: ' + process.cwd())
|
|||
const startHTTPS = true
|
||||
const isRoot = process.getuid && process.getuid() === 0
|
||||
|
||||
const port = 8080
|
||||
const port = process.env.PORT || 8080
|
||||
const httpsport = 5001
|
||||
|
||||
// import os from 'os'
|
||||
|
|
|
|||
21
src/standaloneUtils/serverMaintenenceUtils.js
Normal file
21
src/standaloneUtils/serverMaintenenceUtils.js
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
/* eslint-disable @typescript-eslint/no-var-requires */
|
||||
const { countQuestionsInSubjects } = require('../../dist/utils/qdbUtils.js')
|
||||
const fs = require('fs')
|
||||
|
||||
const command = process.argv[2]
|
||||
const args = process.argv.slice(3)
|
||||
|
||||
const actions = {
|
||||
qdbcount: () => {
|
||||
const qdb = JSON.parse(fs.readFileSync(args[0], 'utf-8'))
|
||||
const questionCount = countQuestionsInSubjects(qdb)
|
||||
console.log({ questionCount: questionCount })
|
||||
},
|
||||
}
|
||||
|
||||
if (actions[command]) {
|
||||
actions[command]()
|
||||
} else {
|
||||
console.log('No action for ' + command)
|
||||
console.log('Possible commands: ', Object.keys(actions))
|
||||
}
|
||||
|
|
@ -1,11 +1,11 @@
|
|||
import { Subject, Question } from '../types/basicTypes'
|
||||
import fs from 'fs'
|
||||
import { RecievedData } from '../utils/actions'
|
||||
import {
|
||||
addQuestion,
|
||||
createQuestion,
|
||||
getSubjNameWithoutYear,
|
||||
} from '../utils/classes'
|
||||
import { Subject, Question } from '../types/basicTypes'
|
||||
import fs from 'fs'
|
||||
import { RecievedData } from '../utils/actions'
|
||||
} from '../utils/qdbUtils'
|
||||
|
||||
const question: Question = createQuestion('asd', 'asd', { type: 'simple' })
|
||||
|
||||
|
|
|
|||
|
|
@ -1,58 +1,23 @@
|
|||
import { updateQuestionsInArray } from '../utils/actions'
|
||||
import { createQuestion } from '../utils/classes'
|
||||
import { cleanDb } from '../utils/classes'
|
||||
import { QuestionDb, Subject, Question } from '../types/basicTypes'
|
||||
|
||||
const date = (x?: number) => new Date().getTime() + (x || 0)
|
||||
import { questions } from './testData'
|
||||
import { cleanDb } from '../utils/qdbUtils'
|
||||
|
||||
const q1 = createQuestion(
|
||||
'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE',
|
||||
'piaci áruk eltérhet a névértéktől.',
|
||||
{
|
||||
type: 'simple',
|
||||
date: date(-1000),
|
||||
}
|
||||
)
|
||||
const q2 = createQuestion(
|
||||
'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE',
|
||||
'afjléa gféda gfdjs légf',
|
||||
{
|
||||
type: 'simple',
|
||||
date: date(-1000),
|
||||
}
|
||||
)
|
||||
const q3 = createQuestion(
|
||||
'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE',
|
||||
'afjlsd gfds dgfs gf sdgf d',
|
||||
{
|
||||
type: 'simple',
|
||||
date: date(-1000),
|
||||
}
|
||||
)
|
||||
const q4 = createQuestion(
|
||||
'A kötvény névértéke',
|
||||
'A kötvényen feltüntetett meghatározott nagyságú összeg.',
|
||||
{
|
||||
type: 'simple',
|
||||
date: date(-1000),
|
||||
}
|
||||
)
|
||||
const q5 = createQuestion(
|
||||
'Mi az osztalék? asd asd',
|
||||
'A vállalati profit egy része..',
|
||||
{
|
||||
type: 'simple',
|
||||
date: date(1000),
|
||||
}
|
||||
)
|
||||
const q6 = createQuestion(
|
||||
'valaim nagyon értelmes kérdés asd asd',
|
||||
'A vállalati profit egy része..',
|
||||
{
|
||||
type: 'simple',
|
||||
date: date(1000),
|
||||
}
|
||||
)
|
||||
const [q1, q2, q3, q4] = questions.slice(0, 4).map((q) => ({
|
||||
...q,
|
||||
data: {
|
||||
...q.data,
|
||||
date: 100,
|
||||
},
|
||||
}))
|
||||
const [q5, q6] = questions.slice(4, 6).map((q) => ({
|
||||
...q,
|
||||
data: {
|
||||
...q.data,
|
||||
date: 1000,
|
||||
},
|
||||
}))
|
||||
|
||||
function setupTest({
|
||||
newQuestions,
|
||||
|
|
@ -68,12 +33,12 @@ function setupTest({
|
|||
...x,
|
||||
data: {
|
||||
...x.data,
|
||||
date: date(),
|
||||
date: 500,
|
||||
},
|
||||
}
|
||||
})
|
||||
const subjName = subjToClean || 'subject'
|
||||
const overwriteFromDate = date(-100)
|
||||
const overwriteBeforeDate = 400
|
||||
const qdbIndex = 0
|
||||
const qdbs: QuestionDb[] = [
|
||||
{
|
||||
|
|
@ -93,7 +58,7 @@ function setupTest({
|
|||
{
|
||||
questions: recievedQuestions,
|
||||
subjToClean: subjName,
|
||||
overwriteFromDate: overwriteFromDate,
|
||||
overwriteBeforeDate: overwriteBeforeDate,
|
||||
qdbIndex: qdbIndex,
|
||||
},
|
||||
qdbs
|
||||
|
|
@ -108,7 +73,7 @@ function setupTest({
|
|||
return {
|
||||
questionIndexesToRemove: questionIndexesToRemove,
|
||||
updatedQuestions: updatedQuestions,
|
||||
overwriteFromDate: overwriteFromDate,
|
||||
overwriteBeforeDate: overwriteBeforeDate,
|
||||
subjIndex: subjIndex,
|
||||
}
|
||||
}
|
||||
|
|
@ -116,21 +81,16 @@ function setupTest({
|
|||
const s1: Subject = { Name: 'test subject', Questions: [q1, q2, q4, q5] }
|
||||
|
||||
test('Old and duplicate questions should be removed from the database', () => {
|
||||
const { questionIndexesToRemove, updatedQuestions, overwriteFromDate } =
|
||||
setupTest({ newQuestions: [q1, q4, q5], data: [s1] })
|
||||
|
||||
expect(questionIndexesToRemove.length).toBe(3)
|
||||
expect(questionIndexesToRemove[0].length).toBe(2)
|
||||
|
||||
expect(updatedQuestions.length).toBe(3)
|
||||
const toremoveCount = updatedQuestions.filter((question) => {
|
||||
return question.Q.includes('TOREMOVE')
|
||||
}).length
|
||||
expect(toremoveCount).toBe(1)
|
||||
const newQuestion = updatedQuestions.find((question) => {
|
||||
return question.Q.includes('TOREMOVE')
|
||||
const { questionIndexesToRemove, updatedQuestions } = setupTest({
|
||||
newQuestions: [q1, q2, q3],
|
||||
data: [s1],
|
||||
})
|
||||
expect(newQuestion.data.date > overwriteFromDate).toBeTruthy()
|
||||
|
||||
expect(questionIndexesToRemove[0].length).toBe(2)
|
||||
expect(questionIndexesToRemove[1].length).toBe(2)
|
||||
expect(questionIndexesToRemove[2].length).toBe(2)
|
||||
|
||||
expect(updatedQuestions.length).toBe(5)
|
||||
})
|
||||
|
||||
const s2: Subject = {
|
||||
|
|
@ -139,30 +99,37 @@ const s2: Subject = {
|
|||
}
|
||||
|
||||
test('Old and duplicate questions should be removed from the database round 2', () => {
|
||||
const { questionIndexesToRemove, updatedQuestions, overwriteFromDate } =
|
||||
setupTest({ newQuestions: [q1, q4, q5], data: [s2] })
|
||||
const { questionIndexesToRemove, updatedQuestions } = setupTest({
|
||||
newQuestions: [q1, q4, q5],
|
||||
data: [s2],
|
||||
})
|
||||
|
||||
expect(questionIndexesToRemove.length).toBe(3)
|
||||
expect(questionIndexesToRemove[0].length).toBe(3)
|
||||
expect(questionIndexesToRemove[1].length).toBe(1)
|
||||
expect(questionIndexesToRemove[2].length).toBe(0)
|
||||
|
||||
expect(updatedQuestions.length).toBe(4)
|
||||
const toremoveCount = updatedQuestions.filter((question) => {
|
||||
return question.Q.includes('TOREMOVE')
|
||||
}).length
|
||||
expect(toremoveCount).toBe(1)
|
||||
const newQuestion = updatedQuestions.find((question) => {
|
||||
return question.Q.includes('TOREMOVE')
|
||||
})
|
||||
|
||||
test('Old and duplicate questions should be removed from the database round 3', () => {
|
||||
const { questionIndexesToRemove, updatedQuestions } = setupTest({
|
||||
newQuestions: [q5, q6],
|
||||
data: [s2],
|
||||
})
|
||||
expect(newQuestion.data.date > overwriteFromDate).toBeTruthy()
|
||||
|
||||
expect(questionIndexesToRemove[0].length).toBe(0)
|
||||
expect(questionIndexesToRemove[1].length).toBe(0)
|
||||
|
||||
expect(updatedQuestions.length).toBe(6)
|
||||
})
|
||||
|
||||
const s3: Subject = {
|
||||
Name: 'test subject',
|
||||
Questions: [q5, q6].map((x) => ({
|
||||
...x,
|
||||
Questions: [q5, q6].map((q) => ({
|
||||
...q,
|
||||
data: {
|
||||
...x.data,
|
||||
date: date(+50000),
|
||||
...q.data,
|
||||
date: 50000,
|
||||
},
|
||||
})),
|
||||
}
|
||||
|
|
|
|||
204
src/tests/p2p.test.ts
Normal file
204
src/tests/p2p.test.ts
Normal file
|
|
@ -0,0 +1,204 @@
|
|||
import { getNewDataSince, mergeSubjects } from '../modules/api/submodules/p2p'
|
||||
import {
|
||||
countQuestionsInSubjects,
|
||||
getSubjectDifference,
|
||||
} from '../utils/qdbUtils'
|
||||
// import { QuestionDb, Subject } from '../types/basicTypes'
|
||||
|
||||
import { questions, subjects } from './testData'
|
||||
|
||||
// ------------------------------------------------------------------------------------
|
||||
// getSubjectDifference
|
||||
// ------------------------------------------------------------------------------------
|
||||
|
||||
test('Merging two similar question dbs should result in the same', () => {
|
||||
const { newData, newSubjects } = getSubjectDifference(subjects, subjects)
|
||||
|
||||
const newQuestionCount = countQuestionsInSubjects(newData)
|
||||
|
||||
expect(newSubjects.length).toBe(0)
|
||||
expect(newQuestionCount).toBe(0)
|
||||
})
|
||||
|
||||
test('Merging qdb with another that has one extra question should get added', () => {
|
||||
const { newData, newSubjects } = getSubjectDifference(subjects, [
|
||||
subjects[0],
|
||||
{
|
||||
...subjects[1],
|
||||
Questions: [...subjects[1].Questions, questions[10]],
|
||||
},
|
||||
])
|
||||
|
||||
const newQuestionCount = countQuestionsInSubjects(newData)
|
||||
|
||||
expect(newSubjects.length).toBe(0)
|
||||
expect(newQuestionCount).toBe(1)
|
||||
})
|
||||
|
||||
test('Qdb merging adding new subject', () => {
|
||||
const { newData, newSubjects } = getSubjectDifference(
|
||||
[subjects[0]],
|
||||
[subjects[0], subjects[1]]
|
||||
)
|
||||
|
||||
const newQuestionCount = countQuestionsInSubjects(newData)
|
||||
|
||||
expect(newSubjects.length).toBe(1)
|
||||
expect(newQuestionCount).toBe(0)
|
||||
})
|
||||
|
||||
test('Qdb merging adding new subject and new questions', () => {
|
||||
const { newData, newSubjects } = getSubjectDifference(
|
||||
[subjects[0]],
|
||||
[
|
||||
{
|
||||
...subjects[0],
|
||||
Questions: [...subjects[0].Questions, questions[10]],
|
||||
},
|
||||
subjects[1],
|
||||
subjects[2],
|
||||
]
|
||||
)
|
||||
|
||||
const newQuestionCount = countQuestionsInSubjects(newData)
|
||||
|
||||
expect(newSubjects.length).toBe(2)
|
||||
expect(newQuestionCount).toBe(1)
|
||||
})
|
||||
|
||||
// ------------------------------------------------------------------------------------
|
||||
// getNewDataSince
|
||||
// ------------------------------------------------------------------------------------
|
||||
|
||||
test('get new data since works', () => {
|
||||
const [q1, q2, q3] = questions
|
||||
.slice(0, 3)
|
||||
.map((q) => ({ ...q, data: { ...q.data, date: 500 } }))
|
||||
const [q4, q5, q6] = questions
|
||||
.slice(3, 6)
|
||||
.map((q) => ({ ...q, data: { ...q.data, date: 1000 } }))
|
||||
|
||||
const res = getNewDataSince(
|
||||
[
|
||||
{
|
||||
Name: '1',
|
||||
Questions: [q1, q2, q3, q4, q5, q6],
|
||||
},
|
||||
],
|
||||
750
|
||||
)
|
||||
expect(res.length).toBe(1)
|
||||
expect(res[0].Questions.length).toBe(3)
|
||||
})
|
||||
|
||||
test('get new data since works, multiple subjects', () => {
|
||||
const [q1, q2, q3] = questions
|
||||
.slice(0, 3)
|
||||
.map((q) => ({ ...q, data: { ...q.data, date: 500 } }))
|
||||
const [q4, q5, q6] = questions
|
||||
.slice(3, 6)
|
||||
.map((q) => ({ ...q, data: { ...q.data, date: 1000 } }))
|
||||
|
||||
const res = getNewDataSince(
|
||||
[
|
||||
{
|
||||
Name: '1',
|
||||
Questions: [q1, q2, q3, q4, q5, q6],
|
||||
},
|
||||
{
|
||||
Name: '2',
|
||||
Questions: [q1, q2, q3, q4],
|
||||
},
|
||||
],
|
||||
750
|
||||
)
|
||||
expect(res.length).toBe(2)
|
||||
expect(res[0].Questions.length).toBe(3)
|
||||
expect(res[1].Questions.length).toBe(1)
|
||||
})
|
||||
|
||||
test('get new data since works, multiple subjects round 2', () => {
|
||||
const [q1, q2, q3] = questions
|
||||
.slice(0, 3)
|
||||
.map((q) => ({ ...q, data: { ...q.data, date: 500 } }))
|
||||
const [q4, q5, q6] = questions
|
||||
.slice(3, 6)
|
||||
.map((q) => ({ ...q, data: { ...q.data, date: 1000 } }))
|
||||
const [q7, q8, q9] = questions
|
||||
.slice(6, 9)
|
||||
.map((q) => ({ ...q, data: { ...q.data, date: 500 } }))
|
||||
|
||||
const res = getNewDataSince(
|
||||
[
|
||||
{
|
||||
Name: '1',
|
||||
Questions: [q1, q2, q3, q4, q5, q6],
|
||||
},
|
||||
{
|
||||
Name: '2',
|
||||
Questions: [q7, q8, q9],
|
||||
},
|
||||
],
|
||||
750
|
||||
)
|
||||
expect(res.length).toBe(1)
|
||||
expect(res[0].Questions.length).toBe(3)
|
||||
})
|
||||
|
||||
// ------------------------------------------------------------------------------------
|
||||
// mergeQdbs
|
||||
// ------------------------------------------------------------------------------------
|
||||
test('merge subjects works', () => {
|
||||
const [s1] = subjects
|
||||
|
||||
const res = mergeSubjects([s1], [s1], [s1])
|
||||
expect(res.length).toBe(2)
|
||||
expect(res[0].Questions.length).toBe(8)
|
||||
})
|
||||
|
||||
test('merge subjects works, three new subjects', () => {
|
||||
const [s1] = subjects
|
||||
|
||||
const res = mergeSubjects([s1], [s1], [s1, s1, s1])
|
||||
expect(res.length).toBe(4)
|
||||
expect(res[0].Questions.length).toBe(8)
|
||||
expect(res[1].Questions.length).toBe(4)
|
||||
expect(res[2].Questions.length).toBe(4)
|
||||
expect(res[3].Questions.length).toBe(4)
|
||||
})
|
||||
|
||||
test('merge subjects works, no new subjects, two subjects to merge', () => {
|
||||
const [s1] = subjects
|
||||
|
||||
const res = mergeSubjects([s1], [s1, s1, s1, s1], [])
|
||||
expect(res.length).toBe(1)
|
||||
expect(res[0].Questions.length).toBe(20)
|
||||
})
|
||||
|
||||
test('merge subjects works, merging a subject with different name gets ignored', () => {
|
||||
const [s1, s2, s3] = subjects
|
||||
|
||||
const res = mergeSubjects([s1, s2], [s3], [])
|
||||
expect(res.length).toBe(2)
|
||||
expect(res[0].Questions.length).toBe(4)
|
||||
expect(res[1].Questions.length).toBe(4)
|
||||
})
|
||||
|
||||
test('merge subjects works, 2 subjects to 2, 1 new', () => {
|
||||
const [s1, s2, s3] = subjects
|
||||
|
||||
const res = mergeSubjects([s1, s2], [s1, s2], [s3])
|
||||
expect(res.length).toBe(3)
|
||||
expect(res[0].Questions.length).toBe(8)
|
||||
expect(res[1].Questions.length).toBe(8)
|
||||
expect(res[2].Questions.length).toBe(4)
|
||||
})
|
||||
|
||||
test('merge subjects works, no new data', () => {
|
||||
const [s1, s2] = subjects
|
||||
|
||||
const res = mergeSubjects([s1, s2], [], [])
|
||||
expect(res.length).toBe(2)
|
||||
expect(res[0].Questions.length).toBe(4)
|
||||
expect(res[1].Questions.length).toBe(4)
|
||||
})
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
import {
|
||||
setNoPossibleAnswersPenalties,
|
||||
SearchResultQuestion,
|
||||
noPossibleAnswerMatchPenalty,
|
||||
} from '../utils/classes'
|
||||
import { setNoPossibleAnswersPenalties } from '../utils/classes'
|
||||
import { Question } from '../types/basicTypes'
|
||||
import {
|
||||
noPossibleAnswerMatchPenalty,
|
||||
SearchResultQuestion,
|
||||
} from '../utils/qdbUtils'
|
||||
|
||||
const matchPercent = 100
|
||||
|
||||
|
|
|
|||
38
src/tests/qdbUtils.test.ts
Normal file
38
src/tests/qdbUtils.test.ts
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
import { getAvailableQdbIndexes } from '../utils/qdbUtils'
|
||||
import { emptyQdb } from './testData'
|
||||
|
||||
test('getAvailableQdbIndexes works, normal order', () => {
|
||||
const qdbs = [0, 1, 2, 3, 4].map((x) => {
|
||||
return { ...emptyQdb, index: x }
|
||||
})
|
||||
const [index] = getAvailableQdbIndexes(qdbs)
|
||||
|
||||
expect(index).toBe(5)
|
||||
})
|
||||
|
||||
test('getAvailableQdbIndexes works, one missing', () => {
|
||||
const qdbs = [0, 1, 2, 3, 5].map((x) => {
|
||||
return { ...emptyQdb, index: x }
|
||||
})
|
||||
const [index] = getAvailableQdbIndexes(qdbs)
|
||||
|
||||
expect(index).toBe(4)
|
||||
})
|
||||
|
||||
test('getAvailableQdbIndexes works, empty qdb', () => {
|
||||
const [index] = getAvailableQdbIndexes([])
|
||||
|
||||
expect(index).toBe(0)
|
||||
})
|
||||
|
||||
test('getAvailableQdbIndexes works, multiple count', () => {
|
||||
const qdbs = [0, 1, 2, 3, 5].map((x) => {
|
||||
return { ...emptyQdb, index: x }
|
||||
})
|
||||
let indexes = getAvailableQdbIndexes(qdbs)
|
||||
expect(indexes.length).toBe(1)
|
||||
expect(indexes[0]).toBe(4)
|
||||
|
||||
indexes = getAvailableQdbIndexes(qdbs, 5)
|
||||
expect(indexes).toStrictEqual([4, 6, 7, 8, 9])
|
||||
})
|
||||
263
src/tests/testData.ts
Normal file
263
src/tests/testData.ts
Normal file
|
|
@ -0,0 +1,263 @@
|
|||
import { QuestionDb } from '../types/basicTypes'
|
||||
import { createQuestion } from '../utils/qdbUtils'
|
||||
|
||||
const rawQuestions = [
|
||||
// 0
|
||||
{
|
||||
Q: 'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE',
|
||||
A: 'piaci áruk eltérhet a névértéktől.',
|
||||
data: {
|
||||
type: 'simple',
|
||||
date: 1678692844547,
|
||||
},
|
||||
},
|
||||
// 1
|
||||
{
|
||||
Q: 'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE',
|
||||
A: 'afjléa gféda gfdjs légf',
|
||||
data: {
|
||||
type: 'simple',
|
||||
date: 1678692844547,
|
||||
},
|
||||
},
|
||||
// 2
|
||||
{
|
||||
Q: 'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE',
|
||||
A: 'afjlsd gfds dgfs gf sdgf d',
|
||||
data: {
|
||||
type: 'simple',
|
||||
date: 1678692844547,
|
||||
},
|
||||
},
|
||||
// 3
|
||||
{
|
||||
Q: 'A kötvény névértéke',
|
||||
A: 'A kötvényen feltüntetett meghatározott nagyságú összeg.',
|
||||
data: {
|
||||
type: 'simple',
|
||||
date: 1678692844547,
|
||||
},
|
||||
},
|
||||
// 4
|
||||
{
|
||||
Q: 'Mi az osztalék? asd asd',
|
||||
A: 'A vállalati profit egy része..',
|
||||
data: {
|
||||
type: 'simple',
|
||||
date: 1678692844547,
|
||||
},
|
||||
},
|
||||
// 5
|
||||
{
|
||||
Q: 'valaim nagyon értelmes kérdés asd asd',
|
||||
A: 'A vállalati profit egy része..',
|
||||
data: {
|
||||
type: 'simple',
|
||||
date: 1678692844547,
|
||||
},
|
||||
},
|
||||
// 6
|
||||
{
|
||||
Q: 'A kötvény és a részvény közös tulajdonsága, hogy',
|
||||
A: 'piaci áruk eltérhet a névértéktől.',
|
||||
data: {
|
||||
type: 'simple',
|
||||
source: 'script',
|
||||
date: 1252626725558,
|
||||
},
|
||||
},
|
||||
// 7
|
||||
{
|
||||
Q: 'A kötvény és a részvény közös tulajdonsága, hogy',
|
||||
A: 'afjléa gféda gfdjs légf',
|
||||
data: {
|
||||
type: 'simple',
|
||||
source: 'script',
|
||||
date: 1252626725558,
|
||||
},
|
||||
},
|
||||
// 8
|
||||
{
|
||||
Q: 'A kötvény névértéke',
|
||||
A: 'A kötvényen feltüntetett meghatározott nagyságú összeg.',
|
||||
data: {
|
||||
type: 'simple',
|
||||
source: 'script',
|
||||
date: 1252626725558,
|
||||
},
|
||||
},
|
||||
// 9
|
||||
{
|
||||
Q: 'A részvényesnek joga van',
|
||||
A: 'Mind a háromra feljogosít.',
|
||||
data: {
|
||||
type: 'simple',
|
||||
source: 'script',
|
||||
date: 1652636725558,
|
||||
},
|
||||
},
|
||||
// 10
|
||||
{
|
||||
Q: 'Mi az osztalék?',
|
||||
A: 'A vállalati profit egy része..',
|
||||
data: {
|
||||
type: 'simple',
|
||||
source: 'script',
|
||||
date: 1652636725559,
|
||||
},
|
||||
},
|
||||
// 11
|
||||
{
|
||||
Q: 'Az alábbi értékpapírok közül melyik kizárólagos kibocsátója a hitelintézet?',
|
||||
A: 'letéti jegy.',
|
||||
data: {
|
||||
type: 'simple',
|
||||
source: 'script',
|
||||
date: 1652636725559,
|
||||
},
|
||||
},
|
||||
// 12
|
||||
{
|
||||
Q: 'Mely állítás nem igaz a kötvényre?',
|
||||
A: 'az osztalék-számítás módját fel kell tüntetni az értékpapíron.',
|
||||
data: {
|
||||
type: 'simple',
|
||||
source: 'script',
|
||||
date: 1652636725559,
|
||||
},
|
||||
},
|
||||
// 13
|
||||
{
|
||||
Q: 'Mely állítás nem igaz a kötvényre?',
|
||||
A: 'tagsági jogot megtestesítő értékpapír.',
|
||||
data: {
|
||||
type: 'simple',
|
||||
source: 'script',
|
||||
date: 1652636725559,
|
||||
},
|
||||
},
|
||||
{
|
||||
Q: 'Az osztalék közvetlenül nem függ',
|
||||
A: 'a részvénytársaság múltbeli költséggazdálkodásától.',
|
||||
data: {
|
||||
type: 'simple',
|
||||
source: 'script',
|
||||
date: 1652636725560,
|
||||
},
|
||||
},
|
||||
{
|
||||
Q: 'Ha a részvénytársaság az egyik tulajdonosától visszavásárolja a saját részvényeit, akkor ezzel',
|
||||
A: 'átrendezi a vállalat tulajdonosi szerkezetét.',
|
||||
data: {
|
||||
type: 'simple',
|
||||
source: 'script',
|
||||
date: 1652636725560,
|
||||
},
|
||||
},
|
||||
{
|
||||
Q: 'Válassza ki az értékpapírokra vonatkozó helyes megállapítást!',
|
||||
A: 'Vagyoni jogot megtestesítő forgalomképes okirat.',
|
||||
data: {
|
||||
type: 'simple',
|
||||
source: 'script',
|
||||
date: 1652636725560,
|
||||
},
|
||||
},
|
||||
{
|
||||
Q: '10. Válassza ki, hogy mely értékpapírtípus járul hozzá egy vállalat alaptőkéjéhez?',
|
||||
A: 'Részesedési jogot megtestesítő értékpapír.',
|
||||
data: {
|
||||
type: 'simple',
|
||||
source: 'script',
|
||||
date: 1652636725560,
|
||||
},
|
||||
},
|
||||
{
|
||||
Q: 'When does the ~/.bashrc script run automatically?',
|
||||
A: 'When a new terminal window is opened..',
|
||||
data: {
|
||||
type: 'simple',
|
||||
source: 'script',
|
||||
date: 1678692844546,
|
||||
},
|
||||
},
|
||||
{
|
||||
Q: 'A robot is ...',
|
||||
A: '... a complex mechatronic system enabled with electronics, sensors, actuators and software, executing tasks with a certain degree of autonomy. It may be preprogrammed, teleoperated or carrying out computations to make decisions. .',
|
||||
data: {
|
||||
type: 'simple',
|
||||
source: 'script',
|
||||
date: 1678692844546,
|
||||
},
|
||||
},
|
||||
{
|
||||
Q: 'A robot is ...',
|
||||
A: '... some sort of device, which has sensors those sensors the world, does some sort of computation, decides on an action, and then does that action based on the sensory input, which makes some change out in the world, outside its body. .',
|
||||
data: {
|
||||
type: 'simple',
|
||||
source: 'script',
|
||||
date: 1678692844546,
|
||||
},
|
||||
},
|
||||
{
|
||||
Q: 'A robot is ...',
|
||||
A: '... an actuated mechanism programmable in two or more axes with a degree of autonomy, moving within its environment, to perform intended tasks. .',
|
||||
data: {
|
||||
type: 'simple',
|
||||
source: 'script',
|
||||
date: 1678692844546,
|
||||
},
|
||||
},
|
||||
{
|
||||
Q: 'ROS is the abbreviation of ...',
|
||||
A: 'Robot Operating System .',
|
||||
data: {
|
||||
type: 'simple',
|
||||
source: 'script',
|
||||
date: 1678692844546,
|
||||
},
|
||||
},
|
||||
{
|
||||
Q: 'A robot is ...',
|
||||
A: '... a machine—especially one programmable by a computer— capable of carrying out a complex series of actions automatically. Robots can be guided by an external control device or the control may be embedded within. Robots may be constructed on the lines of human form, but most robots are machines designed to perform a task with no regard to their aesthetics. .',
|
||||
data: {
|
||||
type: 'simple',
|
||||
source: 'script',
|
||||
date: 1678692844547,
|
||||
},
|
||||
},
|
||||
{
|
||||
Q: 'Complete the definition of a robot: A robot is a complex mechatronic system enabled with electronics, üres , actuators and software, executing tasks with a certain degree of üres . It may be preprogrammed, üres or carrying out computations to make üres .',
|
||||
A: 'Complete the definition of a robot: A robot is a complex mechatronic system enabled with electronics, [sensors], actuators and software, executing tasks with a certain degree of [autonomy]. It may be preprogrammed, [teleoperated] or carrying out computations to make [decisions].',
|
||||
data: {
|
||||
type: 'simple',
|
||||
source: 'script',
|
||||
date: 1678692844547,
|
||||
},
|
||||
},
|
||||
]
|
||||
export const questions = rawQuestions.map((q) => createQuestion(q))
|
||||
|
||||
export const subjects = [
|
||||
{
|
||||
Name: 'Pénzügyek alapjai',
|
||||
Questions: questions.slice(0, 4),
|
||||
},
|
||||
{
|
||||
Name: 'Programming robots in ROS',
|
||||
Questions: questions.slice(4, 8),
|
||||
},
|
||||
{
|
||||
Name: 'Programming something',
|
||||
Questions: questions.slice(8, 12),
|
||||
},
|
||||
]
|
||||
|
||||
export const emptyQdb: QuestionDb = {
|
||||
index: 0,
|
||||
data: [],
|
||||
path: '',
|
||||
name: '',
|
||||
shouldSearch: '',
|
||||
shouldSave: {},
|
||||
}
|
||||
|
|
@ -19,15 +19,15 @@
|
|||
------------------------------------------------------------------------- */
|
||||
|
||||
import express from 'express'
|
||||
import { SearchResultQuestion } from '../utils/classes'
|
||||
import type { Database } from 'better-sqlite3'
|
||||
import type { Socket as SocketIoSocket } from 'socket.io'
|
||||
import http from 'http'
|
||||
import https from 'https'
|
||||
import { SearchResultQuestion } from '../utils/qdbUtils'
|
||||
|
||||
export interface QuestionData {
|
||||
type: string
|
||||
date?: Date | number
|
||||
date?: number
|
||||
images?: Array<string>
|
||||
hashedImages?: Array<string>
|
||||
possibleAnswers?: Array<{
|
||||
|
|
@ -36,6 +36,7 @@ export interface QuestionData {
|
|||
selectedByUser?: boolean
|
||||
}>
|
||||
base64?: string[]
|
||||
source?: string
|
||||
}
|
||||
|
||||
export interface Question {
|
||||
|
|
@ -59,7 +60,7 @@ export interface DataFile {
|
|||
locked?: Boolean
|
||||
overwrites?: Array<{
|
||||
subjName: string
|
||||
overwriteFromDate: number
|
||||
overwriteBeforeDate: number
|
||||
}>
|
||||
shouldSearch:
|
||||
| string
|
||||
|
|
@ -87,16 +88,16 @@ export interface QuestionDb extends DataFile {
|
|||
export interface User {
|
||||
id: number
|
||||
pw: string
|
||||
created: Date
|
||||
}
|
||||
|
||||
export interface User {
|
||||
id: number
|
||||
pw: string
|
||||
pwRequestCount: number
|
||||
avaiblePWRequests: number
|
||||
oldCID?: number
|
||||
notes?: string
|
||||
loginCount: number
|
||||
avaiblePWRequests: number
|
||||
pwRequestCount: number
|
||||
createdBy: number
|
||||
created: number
|
||||
lastLogin: number
|
||||
lastAccess: number
|
||||
sourceHost?: number
|
||||
}
|
||||
|
||||
export interface Request<T = any> extends express.Request {
|
||||
|
|
@ -111,13 +112,21 @@ export interface Request<T = any> extends express.Request {
|
|||
query: { [key: string]: string }
|
||||
}
|
||||
|
||||
export interface ModuleSpecificData {
|
||||
// TODO: rename to something more meaningfull
|
||||
questionDbs: QuestionDb[]
|
||||
setQuestionDbs: (newVal: QuestionDb[]) => void
|
||||
getQuestionDbs: () => QuestionDb[]
|
||||
dbsFile: string
|
||||
}
|
||||
|
||||
export interface SubmoduleData {
|
||||
app: express.Application
|
||||
url: string
|
||||
publicdirs: Array<string>
|
||||
userDB?: Database
|
||||
nextdir?: string
|
||||
moduleSpecificData?: any
|
||||
moduleSpecificData: ModuleSpecificData
|
||||
httpServer: http.Server
|
||||
httpsServer: https.Server
|
||||
}
|
||||
|
|
@ -159,3 +168,11 @@ export interface ModuleType {
|
|||
export interface Socket extends SocketIoSocket {
|
||||
user: User
|
||||
}
|
||||
|
||||
export interface PeerInfo {
|
||||
name: string
|
||||
host: string
|
||||
port: number
|
||||
publicKey: string
|
||||
lastSync?: Date
|
||||
}
|
||||
|
|
|
|||
5
src/types/typeSchemas.ts
Normal file
5
src/types/typeSchemas.ts
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
export const PeerInfoSchema = {
|
||||
name: { type: 'string' },
|
||||
host: { type: 'string' },
|
||||
port: { type: 'string' },
|
||||
}
|
||||
|
|
@ -22,15 +22,16 @@ const recDataFile = './stats/recdata'
|
|||
const dataLockFile = './data/lockData'
|
||||
|
||||
import logger from '../utils/logger'
|
||||
import {
|
||||
createQuestion,
|
||||
WorkerResult,
|
||||
SearchResultQuestion,
|
||||
} from '../utils/classes'
|
||||
import { WorkerResult } from '../utils/classes'
|
||||
import { doALongTask, msgAllWorker } from './workerPool'
|
||||
import idStats from '../utils/ids'
|
||||
import utils from '../utils/utils'
|
||||
import { addQuestion, getSubjNameWithoutYear } from './classes'
|
||||
import {
|
||||
addQuestion,
|
||||
createQuestion,
|
||||
getSubjNameWithoutYear,
|
||||
SearchResultQuestion,
|
||||
} from '../utils/qdbUtils'
|
||||
|
||||
// types
|
||||
import {
|
||||
|
|
@ -40,6 +41,7 @@ import {
|
|||
User,
|
||||
DataFile,
|
||||
} from '../types/basicTypes'
|
||||
import { countOfQdbs } from './qdbUtils'
|
||||
|
||||
// if a recievend question doesnt match at least this % to any other question in the db it gets
|
||||
// added to db
|
||||
|
|
@ -321,8 +323,11 @@ function runCleanWorker(
|
|||
subjName: string,
|
||||
qdb: QuestionDb
|
||||
) {
|
||||
// FIXME: clean worker should compare images too!
|
||||
// see: classes.ts:1011
|
||||
return
|
||||
if (qdb.overwrites && qdb.overwrites.length) {
|
||||
// check if subject needs to be updated, and qdb has overwriteFromDate
|
||||
// check if subject needs to be updated, and qdb has overwriteBeforeDate
|
||||
const overwrite = qdb.overwrites.find((x) => {
|
||||
return subjName.toLowerCase().includes(x.subjName.toLowerCase())
|
||||
})
|
||||
|
|
@ -343,7 +348,7 @@ function runCleanWorker(
|
|||
data: {
|
||||
questions: recievedQuesitons,
|
||||
subjToClean: subjName,
|
||||
overwriteFromDate: overwrite.overwriteFromDate,
|
||||
overwriteBeforeDate: overwrite.overwriteBeforeDate,
|
||||
qdbIndex: qdb.index,
|
||||
},
|
||||
}).then(({ result: questionIndexesToRemove }) => {
|
||||
|
|
@ -391,15 +396,13 @@ export function updateQuestionsInArray(
|
|||
questions: Question[],
|
||||
newQuestions: Question[]
|
||||
): Question[] {
|
||||
const indexesToRemove = questionIndexesToRemove.reduce((acc, x) => {
|
||||
if (x.length > 1) {
|
||||
return [...acc, ...x]
|
||||
}
|
||||
return acc
|
||||
}, [])
|
||||
if (newQuestions.length !== questionIndexesToRemove.length) {
|
||||
throw new Error('newQuestions length ne questionIndexesToRemove length')
|
||||
}
|
||||
|
||||
const indexesToRemove = questionIndexesToRemove.flat()
|
||||
const newQuestionsToAdd: Question[] = newQuestions.filter((_q, i) => {
|
||||
return questionIndexesToRemove[i].length > 1
|
||||
return questionIndexesToRemove[i].length >= 1
|
||||
})
|
||||
|
||||
return [
|
||||
|
|
@ -407,7 +410,7 @@ export function updateQuestionsInArray(
|
|||
return !indexesToRemove.includes(i)
|
||||
}),
|
||||
...newQuestionsToAdd.map((x) => {
|
||||
x.data.date = new Date()
|
||||
x.data.date = new Date().getTime()
|
||||
return x
|
||||
}),
|
||||
]
|
||||
|
|
@ -500,6 +503,10 @@ export function loadJSON(
|
|||
const dataPath = dataDir + dataFile.path
|
||||
|
||||
if (!utils.FileExists(dataPath)) {
|
||||
logger.Log(
|
||||
`${dataPath} data file did not exist, created empty one!`,
|
||||
'yellowbg'
|
||||
)
|
||||
utils.WriteFile(JSON.stringify([]), dataPath)
|
||||
}
|
||||
|
||||
|
|
@ -520,14 +527,7 @@ export function loadJSON(
|
|||
return acc
|
||||
}, [])
|
||||
|
||||
let subjCount = 0
|
||||
let questionCount = 0
|
||||
res.forEach((qdb) => {
|
||||
subjCount += qdb.data.length
|
||||
qdb.data.forEach((subj) => {
|
||||
questionCount += subj.Questions.length
|
||||
})
|
||||
})
|
||||
const { subjCount, questionCount } = countOfQdbs(res)
|
||||
logger.Log(
|
||||
`Loaded ${subjCount} subjects with ${questionCount} questions from ${res.length} question db-s`,
|
||||
logger.GetColor('green')
|
||||
|
|
@ -543,6 +543,7 @@ export function writeData(data: Array<Subject>, path: string): void {
|
|||
return {
|
||||
Name: subj.Name,
|
||||
Questions: subj.Questions.map((question) => {
|
||||
// removing cache here
|
||||
return {
|
||||
Q: question.Q,
|
||||
A: question.A,
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -93,16 +93,13 @@ function DebugLog(msg: string) {
|
|||
}
|
||||
}
|
||||
|
||||
// FIXME: this might not work: what is col exactly, and how we use AddColumn?
|
||||
function AddColumn(
|
||||
db: Database,
|
||||
table: string,
|
||||
col: { [key: string]: string | number }
|
||||
colName: string,
|
||||
colType: string
|
||||
): RunResult {
|
||||
try {
|
||||
const colName = Object.keys(col)[0]
|
||||
const colType = col.type
|
||||
|
||||
const command = `ALTER TABLE ${table} ADD COLUMN ${colName} ${colType}`
|
||||
const stmt = PrepareStatement(db, command)
|
||||
|
||||
|
|
|
|||
53
src/utils/encryption.ts
Normal file
53
src/utils/encryption.ts
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
|
||||
Question Server
|
||||
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
------------------------------------------------------------------------- */
|
||||
|
||||
import { Crypt, RSA } from 'hybrid-crypto-js'
|
||||
|
||||
const rsa = new RSA()
|
||||
const crypt = new Crypt()
|
||||
|
||||
export const createKeyPair = (): Promise<{
|
||||
publicKey: string
|
||||
privateKey: string
|
||||
}> => {
|
||||
return rsa.generateKeyPairAsync()
|
||||
}
|
||||
|
||||
export const encrypt = (publicKey: string, text: string): string => {
|
||||
return crypt.encrypt(publicKey, text)
|
||||
}
|
||||
|
||||
export const decrypt = (privateKey: string, text: string): string => {
|
||||
return crypt.decrypt(privateKey, text).message
|
||||
}
|
||||
|
||||
export const isKeypairValid = (
|
||||
publicKey: string,
|
||||
privateKey: string
|
||||
): boolean => {
|
||||
const testText = 'nem volt jobb ötletem na'
|
||||
try {
|
||||
const encryptedText = encrypt(publicKey, testText)
|
||||
const decryptedText = decrypt(privateKey, encryptedText)
|
||||
return decryptedText === testText
|
||||
} catch (e) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -389,6 +389,9 @@ function C(color?: string): string {
|
|||
if (color === 'redbg') {
|
||||
return '\x1b[41m'
|
||||
}
|
||||
if (color === 'yellowbg') {
|
||||
return '\x1b[43m\x1b[30m'
|
||||
}
|
||||
if (color === 'bluebg') {
|
||||
return '\x1b[44m'
|
||||
}
|
||||
|
|
@ -416,6 +419,23 @@ function C(color?: string): string {
|
|||
return '\x1b[0m'
|
||||
}
|
||||
|
||||
function logTable(table: (string | number)[][]): void {
|
||||
table.forEach((row, i) => {
|
||||
const rowString: string[] = []
|
||||
row.forEach((cell, j) => {
|
||||
const cellColor = j === 0 || i === 0 ? 'blue' : 'green'
|
||||
let cellVal = ''
|
||||
if (!isNaN(+cell)) {
|
||||
cellVal = cell.toLocaleString()
|
||||
} else {
|
||||
cellVal = cell.toString()
|
||||
}
|
||||
rowString.push(C(cellColor) + cellVal + C())
|
||||
})
|
||||
Log(rowString.join('\t'))
|
||||
})
|
||||
}
|
||||
|
||||
export default {
|
||||
getColoredDateString: getColoredDateString,
|
||||
Log: Log,
|
||||
|
|
@ -431,4 +451,5 @@ export default {
|
|||
logDir: logDir,
|
||||
vlogDir: vlogDir,
|
||||
setLoggingDisabled: setLoggingDisabled,
|
||||
logTable: logTable,
|
||||
}
|
||||
|
|
|
|||
606
src/utils/qdbUtils.ts
Normal file
606
src/utils/qdbUtils.ts
Normal file
|
|
@ -0,0 +1,606 @@
|
|||
import logger from './logger'
|
||||
|
||||
import {
|
||||
Question,
|
||||
QuestionData,
|
||||
QuestionDb,
|
||||
Subject,
|
||||
} from '../types/basicTypes'
|
||||
|
||||
interface DetailedMatch {
|
||||
qMatch: number
|
||||
aMatch: number
|
||||
dMatch: number
|
||||
matchedSubjName: string
|
||||
avg: number
|
||||
}
|
||||
|
||||
export interface SearchResultQuestion {
|
||||
q: Question
|
||||
match: number
|
||||
detailedMatch: DetailedMatch
|
||||
}
|
||||
|
||||
/* Percent minus for length difference */
|
||||
const lengthDiffMultiplier = 10
|
||||
// const commonUselessStringParts = [',', '\\.', ':', '!', '\\+', '\\s*\\.']
|
||||
/* Minimum ammount to consider that two questions match during answering */
|
||||
const minMatchAmmount = 75
|
||||
const magicNumber = 0.7 // same as minMatchAmmount, but /100
|
||||
export const minMatchToNotSearchOtherSubjects = 90
|
||||
/* If all of the results are below this match percent (when only one subject is searched due to
|
||||
* subject name matching) then all subjects are searched for answer */
|
||||
export const noPossibleAnswerMatchPenalty = 5
|
||||
|
||||
const commonUselessAnswerParts = [
|
||||
'A helyes válasz az ',
|
||||
'A helyes válasz a ',
|
||||
'A helyes válaszok: ',
|
||||
'A helyes válaszok:',
|
||||
'A helyes válasz: ',
|
||||
'A helyes válasz:',
|
||||
'The correct answer is:',
|
||||
"'",
|
||||
]
|
||||
|
||||
export function getSubjNameWithoutYear(subjName: string): string {
|
||||
const t = subjName.split(' - ')
|
||||
if (t[0].match(/^[0-9]{4}\/[0-9]{2}\/[0-9]{1}$/i)) {
|
||||
return t[1] || subjName
|
||||
} else {
|
||||
return subjName
|
||||
}
|
||||
}
|
||||
|
||||
function simplifyString(toremove: string): string {
|
||||
return toremove.replace(/\s/g, ' ').replace(/\s+/g, ' ').toLowerCase()
|
||||
}
|
||||
|
||||
function removeStuff(
|
||||
value: string,
|
||||
removableStrings: Array<string>,
|
||||
toReplace?: string
|
||||
): string {
|
||||
removableStrings.forEach((removableString) => {
|
||||
const regex = new RegExp(removableString, 'g')
|
||||
value = value.replace(regex, toReplace || '')
|
||||
})
|
||||
return value
|
||||
}
|
||||
|
||||
// damn nonbreaking space
|
||||
function normalizeSpaces(input: string): string {
|
||||
return input.replace(/\s/g, ' ')
|
||||
}
|
||||
|
||||
function removeUnnecesarySpaces(toremove: string): string {
|
||||
return normalizeSpaces(toremove)
|
||||
.replace(/\s+/g, ' ')
|
||||
.replace(/(\r\n|\n|\r)/gm, '')
|
||||
.trim()
|
||||
}
|
||||
|
||||
export function compareString(
|
||||
s1: string,
|
||||
s2: string,
|
||||
s1cache?: Array<string>,
|
||||
s2cache?: Array<string>
|
||||
): number {
|
||||
const s1a = s1cache || s1.split(' ')
|
||||
const s2a = s2cache || s2.split(' ')
|
||||
|
||||
if (s1 === s2) {
|
||||
return 100
|
||||
}
|
||||
if (!s1a || !s2a) {
|
||||
if (!s1a && !s2a) {
|
||||
return 100
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
if (s1a.length < 0 || s2a.length < 0) {
|
||||
if (s1a.length === 0 && s2a.length === 0) {
|
||||
return 100
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
let match = 0
|
||||
let lastMatchIndex = -2
|
||||
let i = 0
|
||||
|
||||
while (i < s1a.length) {
|
||||
if (match / i < magicNumber) {
|
||||
break
|
||||
}
|
||||
|
||||
const currMatchIndex = s2a.indexOf(s1a[i])
|
||||
if (currMatchIndex !== -1 && lastMatchIndex < currMatchIndex) {
|
||||
match++
|
||||
lastMatchIndex = currMatchIndex
|
||||
}
|
||||
|
||||
i++
|
||||
}
|
||||
|
||||
let percent = Math.round(
|
||||
parseFloat(((match / s1a.length) * 100).toFixed(2))
|
||||
)
|
||||
const lengthDifference = Math.abs(s2a.length - s1a.length)
|
||||
percent -= lengthDifference * lengthDiffMultiplier
|
||||
if (percent < 0) {
|
||||
percent = 0
|
||||
}
|
||||
return percent
|
||||
}
|
||||
|
||||
function answerPreProcessor(value: string): string {
|
||||
if (!value) {
|
||||
return value
|
||||
}
|
||||
|
||||
return removeStuff(value, commonUselessAnswerParts)
|
||||
}
|
||||
|
||||
// 'a. pécsi sör' -> 'pécsi sör'
|
||||
function removeAnswerLetters(value: string): string {
|
||||
if (!value) {
|
||||
return value
|
||||
}
|
||||
|
||||
const val = value.split('. ')
|
||||
if (val[0].length < 2 && val.length > 1) {
|
||||
val.shift()
|
||||
return val.join(' ')
|
||||
} else {
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
function simplifyQA(value: string, mods: Array<Function>): string {
|
||||
if (!value) {
|
||||
return value
|
||||
}
|
||||
|
||||
return mods.reduce((res, fn) => {
|
||||
return fn(res)
|
||||
}, value)
|
||||
}
|
||||
|
||||
function simplifyAnswer(value: string): string {
|
||||
if (!value) {
|
||||
return value
|
||||
}
|
||||
return simplifyQA(value, [
|
||||
removeUnnecesarySpaces,
|
||||
answerPreProcessor,
|
||||
removeAnswerLetters,
|
||||
])
|
||||
}
|
||||
|
||||
export function simplifyQuestion(question: string): string {
|
||||
if (!question) {
|
||||
return question
|
||||
}
|
||||
return simplifyQA(question, [removeUnnecesarySpaces, removeAnswerLetters])
|
||||
}
|
||||
|
||||
function simplifyQuestionObj(question: Question): Question {
|
||||
if (!question) {
|
||||
return question
|
||||
}
|
||||
if (question.Q) {
|
||||
question.Q = simplifyQA(question.Q, [
|
||||
removeUnnecesarySpaces,
|
||||
removeAnswerLetters,
|
||||
])
|
||||
}
|
||||
if (question.A) {
|
||||
question.A = simplifyQA(question.A, [
|
||||
removeUnnecesarySpaces,
|
||||
removeAnswerLetters,
|
||||
])
|
||||
}
|
||||
return question
|
||||
}
|
||||
|
||||
export function createQuestion(
|
||||
question: Question | string,
|
||||
answer?: string,
|
||||
data?: QuestionData
|
||||
): Question {
|
||||
try {
|
||||
if (typeof question === 'string') {
|
||||
return {
|
||||
Q: simplifyQuestion(question),
|
||||
A: answer ? simplifyAnswer(answer) : undefined,
|
||||
data: data,
|
||||
cache: {
|
||||
Q: question ? simplifyString(question).split(' ') : [],
|
||||
A: answer ? simplifyString(answer).split(' ') : [],
|
||||
},
|
||||
}
|
||||
} else {
|
||||
return {
|
||||
...question,
|
||||
cache: {
|
||||
Q: question.Q ? simplifyString(question.Q).split(' ') : [],
|
||||
A: question.A ? simplifyString(question.A).split(' ') : [],
|
||||
},
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
logger.Log('Error creating question', logger.GetColor('redbg'))
|
||||
console.error(question, answer, data)
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function compareImage(data: QuestionData, data2: QuestionData): number {
|
||||
if (data.hashedImages && data2.hashedImages) {
|
||||
return compareString(
|
||||
data.hashedImages.join(' '),
|
||||
data2.hashedImages.join(' '),
|
||||
data.hashedImages,
|
||||
data2.hashedImages
|
||||
)
|
||||
} else if (data.images && data2.images) {
|
||||
return (
|
||||
compareString(
|
||||
data.images.join(' '),
|
||||
data2.images.join(' '),
|
||||
data.images,
|
||||
data2.images
|
||||
) - 10
|
||||
)
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
function compareData(q1: Question, q2: Question): number {
|
||||
try {
|
||||
if (q1.data.type === q2.data.type) {
|
||||
const dataType = q1.data.type
|
||||
if (dataType === 'simple') {
|
||||
return -1
|
||||
} else if (dataType === 'image') {
|
||||
return compareImage(q1.data, q2.data)
|
||||
} else {
|
||||
logger.DebugLog(
|
||||
`Unhandled data type ${dataType}`,
|
||||
'Compare question data',
|
||||
1
|
||||
)
|
||||
logger.DebugLog(q1, 'Compare question data', 2)
|
||||
}
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
} catch (error) {
|
||||
logger.DebugLog('Error comparing data', 'Compare question data', 1)
|
||||
logger.DebugLog(error.message, 'Compare question data', 1)
|
||||
logger.DebugLog(error, 'Compare question data', 2)
|
||||
console.error(error)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
function compareQuestion(q1: Question, q2: Question): number {
|
||||
return compareString(q1.Q, q2.Q, q1.cache.Q, q2.cache.Q)
|
||||
// return compareString(
|
||||
// q1.Q,
|
||||
// q1.Q ? q1.Q.split(' ') : [],
|
||||
// q2.Q,
|
||||
// q2.Q ? q2.Q.split(' ') : []
|
||||
// )
|
||||
}
|
||||
|
||||
function compareAnswer(q1: Question, q2: Question): number {
|
||||
return compareString(q1.A, q2.A, q1.cache.A, q2.cache.A)
|
||||
// return compareString(
|
||||
// q1.A,
|
||||
// q1.A ? q1.A.split(' ') : [],
|
||||
// q2.A,
|
||||
// q2.A ? q2.A.split(' ') : []
|
||||
// )
|
||||
}
|
||||
|
||||
export function compareQuestionObj(
|
||||
q1: Question,
|
||||
_q1subjName: string,
|
||||
q2: Question,
|
||||
q2subjName: string
|
||||
): DetailedMatch {
|
||||
const qMatch = compareQuestion(q1, q2)
|
||||
const aMatch = q2.A ? compareAnswer(q1, q2) : 0
|
||||
// -1 if botth questions are simple
|
||||
const dMatch = compareData(q1, q2)
|
||||
|
||||
let avg = -1
|
||||
if (q2.A) {
|
||||
if (dMatch === -1) {
|
||||
avg = Math.min(qMatch, aMatch)
|
||||
} else {
|
||||
avg = Math.min(qMatch, aMatch, dMatch)
|
||||
}
|
||||
} else {
|
||||
if (dMatch === -1) {
|
||||
avg = qMatch
|
||||
} else {
|
||||
avg = Math.min(qMatch, dMatch)
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
qMatch: qMatch,
|
||||
aMatch: aMatch,
|
||||
dMatch: dMatch,
|
||||
matchedSubjName: q2subjName,
|
||||
avg: avg,
|
||||
}
|
||||
}
|
||||
|
||||
function questionToString(question: Question): string {
|
||||
const { Q, A, data } = question
|
||||
|
||||
if (data.type !== 'simple') {
|
||||
return '?' + Q + '\n!' + A + '\n>' + JSON.stringify(data)
|
||||
} else {
|
||||
return '?' + Q + '\n!' + A
|
||||
}
|
||||
}
|
||||
|
||||
function subjectToString(subj: Subject): string {
|
||||
const { Questions, Name } = subj
|
||||
|
||||
const result: string[] = []
|
||||
Questions.forEach((question) => {
|
||||
result.push(questionToString(question))
|
||||
})
|
||||
|
||||
return '+' + Name + '\n' + result.join('\n')
|
||||
}
|
||||
|
||||
export function addQuestion(
|
||||
data: Array<Subject>,
|
||||
subj: string,
|
||||
question: Question
|
||||
): void {
|
||||
logger.DebugLog('Adding new question with subjName: ' + subj, 'qdb add', 1)
|
||||
logger.DebugLog(question, 'qdb add', 3)
|
||||
|
||||
const i = data.findIndex((subject) => {
|
||||
return (
|
||||
subject.Name &&
|
||||
subj
|
||||
.toLowerCase()
|
||||
.includes(getSubjNameWithoutYear(subject.Name).toLowerCase())
|
||||
)
|
||||
})
|
||||
|
||||
if (i !== -1) {
|
||||
logger.DebugLog('Adding new question to existing subject', 'qdb add', 1)
|
||||
data[i].Questions.push(question)
|
||||
} else {
|
||||
logger.Log(`Creating new subject: "${subj}"`)
|
||||
data.push({
|
||||
Name: subj,
|
||||
Questions: [question],
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export function prepareQuestion(question: Question): Question {
|
||||
return simplifyQuestionObj(createQuestion(question))
|
||||
}
|
||||
|
||||
export function dataToString(data: Array<Subject>): string {
|
||||
const result: string[] = []
|
||||
data.forEach((subj) => {
|
||||
result.push(subjectToString(subj))
|
||||
})
|
||||
return result.join('\n\n')
|
||||
}
|
||||
|
||||
export function countQuestionsInSubject(subject: Subject): number {
|
||||
return subject.Questions.length
|
||||
}
|
||||
|
||||
export function countQuestionsInSubjects(subject: Subject[]): number {
|
||||
let questionCount = 0
|
||||
subject.forEach((subj) => {
|
||||
questionCount += countQuestionsInSubject(subj)
|
||||
})
|
||||
return questionCount
|
||||
}
|
||||
|
||||
export function countOfQdb(qdb: QuestionDb): {
|
||||
subjCount: number
|
||||
questionCount: number
|
||||
} {
|
||||
const subjCount = qdb.data.length
|
||||
const questionCount = countQuestionsInSubjects(qdb.data)
|
||||
|
||||
return { subjCount: subjCount, questionCount: questionCount }
|
||||
}
|
||||
|
||||
export function countOfQdbs(qdbs: QuestionDb[]): {
|
||||
subjCount: number
|
||||
questionCount: number
|
||||
} {
|
||||
let questionCount = 0
|
||||
let subjCount = 0
|
||||
|
||||
qdbs.forEach((qdb) => {
|
||||
const { subjCount: sc, questionCount: qc } = countOfQdb(qdb)
|
||||
questionCount += qc
|
||||
subjCount += sc
|
||||
})
|
||||
|
||||
return { subjCount: subjCount, questionCount: questionCount }
|
||||
}
|
||||
|
||||
export function searchSubject(
|
||||
subj: Subject,
|
||||
question: Question,
|
||||
subjName: string,
|
||||
searchTillMatchPercent?: number
|
||||
): SearchResultQuestion[] {
|
||||
let result: SearchResultQuestion[] = []
|
||||
|
||||
let stopSearch = false
|
||||
let i = subj.Questions.length - 1
|
||||
while (i >= 0 && !stopSearch) {
|
||||
const currentQuestion = subj.Questions[i]
|
||||
const percent = compareQuestionObj(
|
||||
currentQuestion,
|
||||
subjName,
|
||||
question,
|
||||
subj.Name
|
||||
)
|
||||
|
||||
if (percent.avg >= minMatchAmmount) {
|
||||
result.push({
|
||||
q: currentQuestion,
|
||||
match: percent.avg,
|
||||
detailedMatch: percent,
|
||||
})
|
||||
}
|
||||
|
||||
if (searchTillMatchPercent && percent.avg >= searchTillMatchPercent) {
|
||||
stopSearch = true
|
||||
}
|
||||
|
||||
i--
|
||||
}
|
||||
|
||||
result = result.sort((q1, q2) => {
|
||||
if (q1.match < q2.match) {
|
||||
return 1
|
||||
} else if (q1.match > q2.match) {
|
||||
return -1
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
})
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
export function getSubjectDifference(
|
||||
subjects: Subject[],
|
||||
subjectsToMerge: Subject[]
|
||||
): { newData: Subject[]; newSubjects: Subject[] } {
|
||||
const newData: Subject[] = []
|
||||
const newSubjects: Subject[] = []
|
||||
subjectsToMerge.forEach((remoteSubj) => {
|
||||
const localSubj = subjects.find((ls) => ls.Name === remoteSubj.Name)
|
||||
if (!localSubj) {
|
||||
newSubjects.push(remoteSubj)
|
||||
return
|
||||
}
|
||||
const addedQuestions: Question[] = []
|
||||
remoteSubj.Questions.forEach((remoteQuestion) => {
|
||||
const searchResult = searchSubject(
|
||||
localSubj,
|
||||
remoteQuestion,
|
||||
localSubj.Name,
|
||||
95 // FIXME: maybe fine tune
|
||||
)
|
||||
|
||||
if (searchResult.length === 0) {
|
||||
addedQuestions.push(remoteQuestion)
|
||||
}
|
||||
})
|
||||
if (addedQuestions.length > 0) {
|
||||
newData.push({
|
||||
Name: localSubj.Name,
|
||||
Questions: addedQuestions,
|
||||
})
|
||||
}
|
||||
})
|
||||
return { newData: newData, newSubjects: newSubjects }
|
||||
}
|
||||
|
||||
export function cleanDb(
|
||||
{
|
||||
questions: recievedQuestions,
|
||||
subjToClean,
|
||||
overwriteBeforeDate,
|
||||
qdbIndex,
|
||||
}: {
|
||||
questions: Question[]
|
||||
subjToClean: string
|
||||
overwriteBeforeDate: number
|
||||
qdbIndex: number
|
||||
},
|
||||
qdbs: QuestionDb[]
|
||||
): number[][] {
|
||||
const subjIndex = qdbs[qdbIndex].data.findIndex((x) => {
|
||||
return x.Name.toLowerCase().includes(subjToClean.toLowerCase())
|
||||
})
|
||||
|
||||
if (!qdbs[qdbIndex].data[subjIndex]) {
|
||||
return recievedQuestions.map(() => [])
|
||||
}
|
||||
|
||||
// FIXME: compare images & data too!
|
||||
const questionIndexesToRemove = recievedQuestions.map((recievedQuestion) =>
|
||||
qdbs[qdbIndex].data[subjIndex].Questions.reduce<number[]>(
|
||||
(acc, question, i) => {
|
||||
const res = compareString(
|
||||
simplifyQuestion(recievedQuestion.Q),
|
||||
simplifyQuestion(question.Q)
|
||||
)
|
||||
|
||||
if (
|
||||
res > minMatchToNotSearchOtherSubjects &&
|
||||
(!question.data.date ||
|
||||
question.data.date < overwriteBeforeDate)
|
||||
) {
|
||||
// questions indexes in subject, that should be
|
||||
// removed because of recievedQuestion
|
||||
return [...acc, i]
|
||||
}
|
||||
return acc
|
||||
},
|
||||
[]
|
||||
)
|
||||
)
|
||||
|
||||
return questionIndexesToRemove
|
||||
}
|
||||
|
||||
export function removeCacheFromQuestion(question: Question): Question {
|
||||
if (question.cache) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
const { cache, ...questionWithoutCache } = question
|
||||
return questionWithoutCache
|
||||
} else {
|
||||
return question
|
||||
}
|
||||
}
|
||||
|
||||
export function getAvailableQdbIndexes(
|
||||
qdbs: QuestionDb[],
|
||||
count = 1,
|
||||
initialIndex?: number
|
||||
): number[] {
|
||||
const indexes = qdbs.map((x) => x.index)
|
||||
const availableIndexes: number[] = []
|
||||
|
||||
const minCount = count < 1 ? 1 : count
|
||||
|
||||
let i = initialIndex || 0
|
||||
while (availableIndexes.length < minCount) {
|
||||
if (!indexes.includes(i)) {
|
||||
availableIndexes.push(i)
|
||||
}
|
||||
i += 1
|
||||
}
|
||||
return availableIndexes
|
||||
}
|
||||
|
|
@ -36,6 +36,7 @@ interface WorkerObj {
|
|||
free: Boolean
|
||||
}
|
||||
|
||||
// FIXME: type depending on type
|
||||
export interface TaskObject {
|
||||
type:
|
||||
| 'work'
|
||||
|
|
@ -44,6 +45,7 @@ export interface TaskObject {
|
|||
| 'newdb'
|
||||
| 'dbClean'
|
||||
| 'rmQuestions'
|
||||
| 'merge'
|
||||
data:
|
||||
| {
|
||||
searchIn: number[]
|
||||
|
|
@ -57,11 +59,11 @@ export interface TaskObject {
|
|||
}
|
||||
| { dbIndex: number; edits: Edits }
|
||||
| QuestionDb
|
||||
| Result
|
||||
| Omit<Result, 'qdbName'>
|
||||
| {
|
||||
questions: Question[]
|
||||
subjToClean: string
|
||||
overwriteFromDate: number
|
||||
overwriteBeforeDate: number
|
||||
qdbIndex: number
|
||||
}
|
||||
| {
|
||||
|
|
@ -70,6 +72,10 @@ export interface TaskObject {
|
|||
qdbIndex: number
|
||||
recievedQuestions: Question[]
|
||||
}
|
||||
| {
|
||||
localQdbIndex: number
|
||||
remoteQdb: QuestionDb
|
||||
}
|
||||
}
|
||||
|
||||
interface PendingJob {
|
||||
|
|
@ -90,7 +96,7 @@ interface DoneEvent extends EventEmitter {
|
|||
emit(event: 'done', res: WorkerResult): boolean
|
||||
}
|
||||
|
||||
const alertOnPendingCount = 50
|
||||
const alertOnPendingCount = 100
|
||||
const workerFile = './src/utils/classes.ts'
|
||||
let workers: Array<WorkerObj>
|
||||
let getInitData: () => Array<QuestionDb> = null
|
||||
|
|
@ -136,11 +142,10 @@ export function doALongTask(
|
|||
targetWorkerIndex?: number
|
||||
): Promise<WorkerResult> {
|
||||
if (Object.keys(pendingJobs).length > alertOnPendingCount) {
|
||||
logger.Log(
|
||||
console.error(
|
||||
`More than ${alertOnPendingCount} callers waiting for free resource! (${
|
||||
Object.keys(pendingJobs).length
|
||||
})`,
|
||||
logger.GetColor('redbg')
|
||||
})`
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue