mirror of
https://gitlab.com/MrFry/mrfrys-node-server
synced 2025-04-01 20:24:18 +02:00
added nearly complete p2p implementation
This commit is contained in:
mrfry
parent
11dacdae64
commit
5c22f575dd
25 changed files with 14320 additions and 12563 deletions
|
|
@ -22,15 +22,16 @@ const recDataFile = './stats/recdata'
|
|||
const dataLockFile = './data/lockData'
|
||||
|
||||
import logger from '../utils/logger'
|
||||
import {
|
||||
createQuestion,
|
||||
WorkerResult,
|
||||
SearchResultQuestion,
|
||||
} from '../utils/classes'
|
||||
import { WorkerResult } from '../utils/classes'
|
||||
import { doALongTask, msgAllWorker } from './workerPool'
|
||||
import idStats from '../utils/ids'
|
||||
import utils from '../utils/utils'
|
||||
import { addQuestion, getSubjNameWithoutYear } from './classes'
|
||||
import {
|
||||
addQuestion,
|
||||
createQuestion,
|
||||
getSubjNameWithoutYear,
|
||||
SearchResultQuestion,
|
||||
} from '../utils/qdbUtils'
|
||||
|
||||
// types
|
||||
import {
|
||||
|
|
@ -40,6 +41,7 @@ import {
|
|||
User,
|
||||
DataFile,
|
||||
} from '../types/basicTypes'
|
||||
import { countOfQdbs } from './qdbUtils'
|
||||
|
||||
// if a recievend question doesnt match at least this % to any other question in the db it gets
|
||||
// added to db
|
||||
|
|
@ -321,8 +323,11 @@ function runCleanWorker(
|
|||
subjName: string,
|
||||
qdb: QuestionDb
|
||||
) {
|
||||
// FIXME: clean worker should compare images too!
|
||||
// see: classes.ts:1011
|
||||
return
|
||||
if (qdb.overwrites && qdb.overwrites.length) {
|
||||
// check if subject needs to be updated, and qdb has overwriteFromDate
|
||||
// check if subject needs to be updated, and qdb has overwriteBeforeDate
|
||||
const overwrite = qdb.overwrites.find((x) => {
|
||||
return subjName.toLowerCase().includes(x.subjName.toLowerCase())
|
||||
})
|
||||
|
|
@ -343,7 +348,7 @@ function runCleanWorker(
|
|||
data: {
|
||||
questions: recievedQuesitons,
|
||||
subjToClean: subjName,
|
||||
overwriteFromDate: overwrite.overwriteFromDate,
|
||||
overwriteBeforeDate: overwrite.overwriteBeforeDate,
|
||||
qdbIndex: qdb.index,
|
||||
},
|
||||
}).then(({ result: questionIndexesToRemove }) => {
|
||||
|
|
@ -391,15 +396,13 @@ export function updateQuestionsInArray(
|
|||
questions: Question[],
|
||||
newQuestions: Question[]
|
||||
): Question[] {
|
||||
const indexesToRemove = questionIndexesToRemove.reduce((acc, x) => {
|
||||
if (x.length > 1) {
|
||||
return [...acc, ...x]
|
||||
}
|
||||
return acc
|
||||
}, [])
|
||||
if (newQuestions.length !== questionIndexesToRemove.length) {
|
||||
throw new Error('newQuestions length ne questionIndexesToRemove length')
|
||||
}
|
||||
|
||||
const indexesToRemove = questionIndexesToRemove.flat()
|
||||
const newQuestionsToAdd: Question[] = newQuestions.filter((_q, i) => {
|
||||
return questionIndexesToRemove[i].length > 1
|
||||
return questionIndexesToRemove[i].length >= 1
|
||||
})
|
||||
|
||||
return [
|
||||
|
|
@ -407,7 +410,7 @@ export function updateQuestionsInArray(
|
|||
return !indexesToRemove.includes(i)
|
||||
}),
|
||||
...newQuestionsToAdd.map((x) => {
|
||||
x.data.date = new Date()
|
||||
x.data.date = new Date().getTime()
|
||||
return x
|
||||
}),
|
||||
]
|
||||
|
|
@ -500,6 +503,10 @@ export function loadJSON(
|
|||
const dataPath = dataDir + dataFile.path
|
||||
|
||||
if (!utils.FileExists(dataPath)) {
|
||||
logger.Log(
|
||||
`${dataPath} data file did not exist, created empty one!`,
|
||||
'yellowbg'
|
||||
)
|
||||
utils.WriteFile(JSON.stringify([]), dataPath)
|
||||
}
|
||||
|
||||
|
|
@ -520,14 +527,7 @@ export function loadJSON(
|
|||
return acc
|
||||
}, [])
|
||||
|
||||
let subjCount = 0
|
||||
let questionCount = 0
|
||||
res.forEach((qdb) => {
|
||||
subjCount += qdb.data.length
|
||||
qdb.data.forEach((subj) => {
|
||||
questionCount += subj.Questions.length
|
||||
})
|
||||
})
|
||||
const { subjCount, questionCount } = countOfQdbs(res)
|
||||
logger.Log(
|
||||
`Loaded ${subjCount} subjects with ${questionCount} questions from ${res.length} question db-s`,
|
||||
logger.GetColor('green')
|
||||
|
|
@ -543,6 +543,7 @@ export function writeData(data: Array<Subject>, path: string): void {
|
|||
return {
|
||||
Name: subj.Name,
|
||||
Questions: subj.Questions.map((question) => {
|
||||
// removing cache here
|
||||
return {
|
||||
Q: question.Q,
|
||||
A: question.A,
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -93,16 +93,13 @@ function DebugLog(msg: string) {
|
|||
}
|
||||
}
|
||||
|
||||
// FIXME: this might not work: what is col exactly, and how we use AddColumn?
|
||||
function AddColumn(
|
||||
db: Database,
|
||||
table: string,
|
||||
col: { [key: string]: string | number }
|
||||
colName: string,
|
||||
colType: string
|
||||
): RunResult {
|
||||
try {
|
||||
const colName = Object.keys(col)[0]
|
||||
const colType = col.type
|
||||
|
||||
const command = `ALTER TABLE ${table} ADD COLUMN ${colName} ${colType}`
|
||||
const stmt = PrepareStatement(db, command)
|
||||
|
||||
|
|
|
|||
53
src/utils/encryption.ts
Normal file
53
src/utils/encryption.ts
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
|
||||
Question Server
|
||||
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
------------------------------------------------------------------------- */
|
||||
|
||||
import { Crypt, RSA } from 'hybrid-crypto-js'
|
||||
|
||||
const rsa = new RSA()
|
||||
const crypt = new Crypt()
|
||||
|
||||
export const createKeyPair = (): Promise<{
|
||||
publicKey: string
|
||||
privateKey: string
|
||||
}> => {
|
||||
return rsa.generateKeyPairAsync()
|
||||
}
|
||||
|
||||
export const encrypt = (publicKey: string, text: string): string => {
|
||||
return crypt.encrypt(publicKey, text)
|
||||
}
|
||||
|
||||
export const decrypt = (privateKey: string, text: string): string => {
|
||||
return crypt.decrypt(privateKey, text).message
|
||||
}
|
||||
|
||||
export const isKeypairValid = (
|
||||
publicKey: string,
|
||||
privateKey: string
|
||||
): boolean => {
|
||||
const testText = 'nem volt jobb ötletem na'
|
||||
try {
|
||||
const encryptedText = encrypt(publicKey, testText)
|
||||
const decryptedText = decrypt(privateKey, encryptedText)
|
||||
return decryptedText === testText
|
||||
} catch (e) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -389,6 +389,9 @@ function C(color?: string): string {
|
|||
if (color === 'redbg') {
|
||||
return '\x1b[41m'
|
||||
}
|
||||
if (color === 'yellowbg') {
|
||||
return '\x1b[43m\x1b[30m'
|
||||
}
|
||||
if (color === 'bluebg') {
|
||||
return '\x1b[44m'
|
||||
}
|
||||
|
|
@ -416,6 +419,23 @@ function C(color?: string): string {
|
|||
return '\x1b[0m'
|
||||
}
|
||||
|
||||
function logTable(table: (string | number)[][]): void {
|
||||
table.forEach((row, i) => {
|
||||
const rowString: string[] = []
|
||||
row.forEach((cell, j) => {
|
||||
const cellColor = j === 0 || i === 0 ? 'blue' : 'green'
|
||||
let cellVal = ''
|
||||
if (!isNaN(+cell)) {
|
||||
cellVal = cell.toLocaleString()
|
||||
} else {
|
||||
cellVal = cell.toString()
|
||||
}
|
||||
rowString.push(C(cellColor) + cellVal + C())
|
||||
})
|
||||
Log(rowString.join('\t'))
|
||||
})
|
||||
}
|
||||
|
||||
export default {
|
||||
getColoredDateString: getColoredDateString,
|
||||
Log: Log,
|
||||
|
|
@ -431,4 +451,5 @@ export default {
|
|||
logDir: logDir,
|
||||
vlogDir: vlogDir,
|
||||
setLoggingDisabled: setLoggingDisabled,
|
||||
logTable: logTable,
|
||||
}
|
||||
|
|
|
|||
606
src/utils/qdbUtils.ts
Normal file
606
src/utils/qdbUtils.ts
Normal file
|
|
@ -0,0 +1,606 @@
|
|||
import logger from './logger'
|
||||
|
||||
import {
|
||||
Question,
|
||||
QuestionData,
|
||||
QuestionDb,
|
||||
Subject,
|
||||
} from '../types/basicTypes'
|
||||
|
||||
interface DetailedMatch {
|
||||
qMatch: number
|
||||
aMatch: number
|
||||
dMatch: number
|
||||
matchedSubjName: string
|
||||
avg: number
|
||||
}
|
||||
|
||||
export interface SearchResultQuestion {
|
||||
q: Question
|
||||
match: number
|
||||
detailedMatch: DetailedMatch
|
||||
}
|
||||
|
||||
/* Percent minus for length difference */
|
||||
const lengthDiffMultiplier = 10
|
||||
// const commonUselessStringParts = [',', '\\.', ':', '!', '\\+', '\\s*\\.']
|
||||
/* Minimum ammount to consider that two questions match during answering */
|
||||
const minMatchAmmount = 75
|
||||
const magicNumber = 0.7 // same as minMatchAmmount, but /100
|
||||
export const minMatchToNotSearchOtherSubjects = 90
|
||||
/* If all of the results are below this match percent (when only one subject is searched due to
|
||||
* subject name matching) then all subjects are searched for answer */
|
||||
export const noPossibleAnswerMatchPenalty = 5
|
||||
|
||||
const commonUselessAnswerParts = [
|
||||
'A helyes válasz az ',
|
||||
'A helyes válasz a ',
|
||||
'A helyes válaszok: ',
|
||||
'A helyes válaszok:',
|
||||
'A helyes válasz: ',
|
||||
'A helyes válasz:',
|
||||
'The correct answer is:',
|
||||
"'",
|
||||
]
|
||||
|
||||
export function getSubjNameWithoutYear(subjName: string): string {
|
||||
const t = subjName.split(' - ')
|
||||
if (t[0].match(/^[0-9]{4}\/[0-9]{2}\/[0-9]{1}$/i)) {
|
||||
return t[1] || subjName
|
||||
} else {
|
||||
return subjName
|
||||
}
|
||||
}
|
||||
|
||||
function simplifyString(toremove: string): string {
|
||||
return toremove.replace(/\s/g, ' ').replace(/\s+/g, ' ').toLowerCase()
|
||||
}
|
||||
|
||||
function removeStuff(
|
||||
value: string,
|
||||
removableStrings: Array<string>,
|
||||
toReplace?: string
|
||||
): string {
|
||||
removableStrings.forEach((removableString) => {
|
||||
const regex = new RegExp(removableString, 'g')
|
||||
value = value.replace(regex, toReplace || '')
|
||||
})
|
||||
return value
|
||||
}
|
||||
|
||||
// damn nonbreaking space
|
||||
function normalizeSpaces(input: string): string {
|
||||
return input.replace(/\s/g, ' ')
|
||||
}
|
||||
|
||||
function removeUnnecesarySpaces(toremove: string): string {
|
||||
return normalizeSpaces(toremove)
|
||||
.replace(/\s+/g, ' ')
|
||||
.replace(/(\r\n|\n|\r)/gm, '')
|
||||
.trim()
|
||||
}
|
||||
|
||||
export function compareString(
|
||||
s1: string,
|
||||
s2: string,
|
||||
s1cache?: Array<string>,
|
||||
s2cache?: Array<string>
|
||||
): number {
|
||||
const s1a = s1cache || s1.split(' ')
|
||||
const s2a = s2cache || s2.split(' ')
|
||||
|
||||
if (s1 === s2) {
|
||||
return 100
|
||||
}
|
||||
if (!s1a || !s2a) {
|
||||
if (!s1a && !s2a) {
|
||||
return 100
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
if (s1a.length < 0 || s2a.length < 0) {
|
||||
if (s1a.length === 0 && s2a.length === 0) {
|
||||
return 100
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
let match = 0
|
||||
let lastMatchIndex = -2
|
||||
let i = 0
|
||||
|
||||
while (i < s1a.length) {
|
||||
if (match / i < magicNumber) {
|
||||
break
|
||||
}
|
||||
|
||||
const currMatchIndex = s2a.indexOf(s1a[i])
|
||||
if (currMatchIndex !== -1 && lastMatchIndex < currMatchIndex) {
|
||||
match++
|
||||
lastMatchIndex = currMatchIndex
|
||||
}
|
||||
|
||||
i++
|
||||
}
|
||||
|
||||
let percent = Math.round(
|
||||
parseFloat(((match / s1a.length) * 100).toFixed(2))
|
||||
)
|
||||
const lengthDifference = Math.abs(s2a.length - s1a.length)
|
||||
percent -= lengthDifference * lengthDiffMultiplier
|
||||
if (percent < 0) {
|
||||
percent = 0
|
||||
}
|
||||
return percent
|
||||
}
|
||||
|
||||
function answerPreProcessor(value: string): string {
|
||||
if (!value) {
|
||||
return value
|
||||
}
|
||||
|
||||
return removeStuff(value, commonUselessAnswerParts)
|
||||
}
|
||||
|
||||
// 'a. pécsi sör' -> 'pécsi sör'
|
||||
function removeAnswerLetters(value: string): string {
|
||||
if (!value) {
|
||||
return value
|
||||
}
|
||||
|
||||
const val = value.split('. ')
|
||||
if (val[0].length < 2 && val.length > 1) {
|
||||
val.shift()
|
||||
return val.join(' ')
|
||||
} else {
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
function simplifyQA(value: string, mods: Array<Function>): string {
|
||||
if (!value) {
|
||||
return value
|
||||
}
|
||||
|
||||
return mods.reduce((res, fn) => {
|
||||
return fn(res)
|
||||
}, value)
|
||||
}
|
||||
|
||||
function simplifyAnswer(value: string): string {
|
||||
if (!value) {
|
||||
return value
|
||||
}
|
||||
return simplifyQA(value, [
|
||||
removeUnnecesarySpaces,
|
||||
answerPreProcessor,
|
||||
removeAnswerLetters,
|
||||
])
|
||||
}
|
||||
|
||||
export function simplifyQuestion(question: string): string {
|
||||
if (!question) {
|
||||
return question
|
||||
}
|
||||
return simplifyQA(question, [removeUnnecesarySpaces, removeAnswerLetters])
|
||||
}
|
||||
|
||||
function simplifyQuestionObj(question: Question): Question {
|
||||
if (!question) {
|
||||
return question
|
||||
}
|
||||
if (question.Q) {
|
||||
question.Q = simplifyQA(question.Q, [
|
||||
removeUnnecesarySpaces,
|
||||
removeAnswerLetters,
|
||||
])
|
||||
}
|
||||
if (question.A) {
|
||||
question.A = simplifyQA(question.A, [
|
||||
removeUnnecesarySpaces,
|
||||
removeAnswerLetters,
|
||||
])
|
||||
}
|
||||
return question
|
||||
}
|
||||
|
||||
export function createQuestion(
|
||||
question: Question | string,
|
||||
answer?: string,
|
||||
data?: QuestionData
|
||||
): Question {
|
||||
try {
|
||||
if (typeof question === 'string') {
|
||||
return {
|
||||
Q: simplifyQuestion(question),
|
||||
A: answer ? simplifyAnswer(answer) : undefined,
|
||||
data: data,
|
||||
cache: {
|
||||
Q: question ? simplifyString(question).split(' ') : [],
|
||||
A: answer ? simplifyString(answer).split(' ') : [],
|
||||
},
|
||||
}
|
||||
} else {
|
||||
return {
|
||||
...question,
|
||||
cache: {
|
||||
Q: question.Q ? simplifyString(question.Q).split(' ') : [],
|
||||
A: question.A ? simplifyString(question.A).split(' ') : [],
|
||||
},
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
logger.Log('Error creating question', logger.GetColor('redbg'))
|
||||
console.error(question, answer, data)
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function compareImage(data: QuestionData, data2: QuestionData): number {
|
||||
if (data.hashedImages && data2.hashedImages) {
|
||||
return compareString(
|
||||
data.hashedImages.join(' '),
|
||||
data2.hashedImages.join(' '),
|
||||
data.hashedImages,
|
||||
data2.hashedImages
|
||||
)
|
||||
} else if (data.images && data2.images) {
|
||||
return (
|
||||
compareString(
|
||||
data.images.join(' '),
|
||||
data2.images.join(' '),
|
||||
data.images,
|
||||
data2.images
|
||||
) - 10
|
||||
)
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
function compareData(q1: Question, q2: Question): number {
|
||||
try {
|
||||
if (q1.data.type === q2.data.type) {
|
||||
const dataType = q1.data.type
|
||||
if (dataType === 'simple') {
|
||||
return -1
|
||||
} else if (dataType === 'image') {
|
||||
return compareImage(q1.data, q2.data)
|
||||
} else {
|
||||
logger.DebugLog(
|
||||
`Unhandled data type ${dataType}`,
|
||||
'Compare question data',
|
||||
1
|
||||
)
|
||||
logger.DebugLog(q1, 'Compare question data', 2)
|
||||
}
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
} catch (error) {
|
||||
logger.DebugLog('Error comparing data', 'Compare question data', 1)
|
||||
logger.DebugLog(error.message, 'Compare question data', 1)
|
||||
logger.DebugLog(error, 'Compare question data', 2)
|
||||
console.error(error)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
function compareQuestion(q1: Question, q2: Question): number {
|
||||
return compareString(q1.Q, q2.Q, q1.cache.Q, q2.cache.Q)
|
||||
// return compareString(
|
||||
// q1.Q,
|
||||
// q1.Q ? q1.Q.split(' ') : [],
|
||||
// q2.Q,
|
||||
// q2.Q ? q2.Q.split(' ') : []
|
||||
// )
|
||||
}
|
||||
|
||||
function compareAnswer(q1: Question, q2: Question): number {
|
||||
return compareString(q1.A, q2.A, q1.cache.A, q2.cache.A)
|
||||
// return compareString(
|
||||
// q1.A,
|
||||
// q1.A ? q1.A.split(' ') : [],
|
||||
// q2.A,
|
||||
// q2.A ? q2.A.split(' ') : []
|
||||
// )
|
||||
}
|
||||
|
||||
export function compareQuestionObj(
|
||||
q1: Question,
|
||||
_q1subjName: string,
|
||||
q2: Question,
|
||||
q2subjName: string
|
||||
): DetailedMatch {
|
||||
const qMatch = compareQuestion(q1, q2)
|
||||
const aMatch = q2.A ? compareAnswer(q1, q2) : 0
|
||||
// -1 if botth questions are simple
|
||||
const dMatch = compareData(q1, q2)
|
||||
|
||||
let avg = -1
|
||||
if (q2.A) {
|
||||
if (dMatch === -1) {
|
||||
avg = Math.min(qMatch, aMatch)
|
||||
} else {
|
||||
avg = Math.min(qMatch, aMatch, dMatch)
|
||||
}
|
||||
} else {
|
||||
if (dMatch === -1) {
|
||||
avg = qMatch
|
||||
} else {
|
||||
avg = Math.min(qMatch, dMatch)
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
qMatch: qMatch,
|
||||
aMatch: aMatch,
|
||||
dMatch: dMatch,
|
||||
matchedSubjName: q2subjName,
|
||||
avg: avg,
|
||||
}
|
||||
}
|
||||
|
||||
function questionToString(question: Question): string {
|
||||
const { Q, A, data } = question
|
||||
|
||||
if (data.type !== 'simple') {
|
||||
return '?' + Q + '\n!' + A + '\n>' + JSON.stringify(data)
|
||||
} else {
|
||||
return '?' + Q + '\n!' + A
|
||||
}
|
||||
}
|
||||
|
||||
function subjectToString(subj: Subject): string {
|
||||
const { Questions, Name } = subj
|
||||
|
||||
const result: string[] = []
|
||||
Questions.forEach((question) => {
|
||||
result.push(questionToString(question))
|
||||
})
|
||||
|
||||
return '+' + Name + '\n' + result.join('\n')
|
||||
}
|
||||
|
||||
export function addQuestion(
|
||||
data: Array<Subject>,
|
||||
subj: string,
|
||||
question: Question
|
||||
): void {
|
||||
logger.DebugLog('Adding new question with subjName: ' + subj, 'qdb add', 1)
|
||||
logger.DebugLog(question, 'qdb add', 3)
|
||||
|
||||
const i = data.findIndex((subject) => {
|
||||
return (
|
||||
subject.Name &&
|
||||
subj
|
||||
.toLowerCase()
|
||||
.includes(getSubjNameWithoutYear(subject.Name).toLowerCase())
|
||||
)
|
||||
})
|
||||
|
||||
if (i !== -1) {
|
||||
logger.DebugLog('Adding new question to existing subject', 'qdb add', 1)
|
||||
data[i].Questions.push(question)
|
||||
} else {
|
||||
logger.Log(`Creating new subject: "${subj}"`)
|
||||
data.push({
|
||||
Name: subj,
|
||||
Questions: [question],
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export function prepareQuestion(question: Question): Question {
|
||||
return simplifyQuestionObj(createQuestion(question))
|
||||
}
|
||||
|
||||
export function dataToString(data: Array<Subject>): string {
|
||||
const result: string[] = []
|
||||
data.forEach((subj) => {
|
||||
result.push(subjectToString(subj))
|
||||
})
|
||||
return result.join('\n\n')
|
||||
}
|
||||
|
||||
export function countQuestionsInSubject(subject: Subject): number {
|
||||
return subject.Questions.length
|
||||
}
|
||||
|
||||
export function countQuestionsInSubjects(subject: Subject[]): number {
|
||||
let questionCount = 0
|
||||
subject.forEach((subj) => {
|
||||
questionCount += countQuestionsInSubject(subj)
|
||||
})
|
||||
return questionCount
|
||||
}
|
||||
|
||||
export function countOfQdb(qdb: QuestionDb): {
|
||||
subjCount: number
|
||||
questionCount: number
|
||||
} {
|
||||
const subjCount = qdb.data.length
|
||||
const questionCount = countQuestionsInSubjects(qdb.data)
|
||||
|
||||
return { subjCount: subjCount, questionCount: questionCount }
|
||||
}
|
||||
|
||||
export function countOfQdbs(qdbs: QuestionDb[]): {
|
||||
subjCount: number
|
||||
questionCount: number
|
||||
} {
|
||||
let questionCount = 0
|
||||
let subjCount = 0
|
||||
|
||||
qdbs.forEach((qdb) => {
|
||||
const { subjCount: sc, questionCount: qc } = countOfQdb(qdb)
|
||||
questionCount += qc
|
||||
subjCount += sc
|
||||
})
|
||||
|
||||
return { subjCount: subjCount, questionCount: questionCount }
|
||||
}
|
||||
|
||||
export function searchSubject(
|
||||
subj: Subject,
|
||||
question: Question,
|
||||
subjName: string,
|
||||
searchTillMatchPercent?: number
|
||||
): SearchResultQuestion[] {
|
||||
let result: SearchResultQuestion[] = []
|
||||
|
||||
let stopSearch = false
|
||||
let i = subj.Questions.length - 1
|
||||
while (i >= 0 && !stopSearch) {
|
||||
const currentQuestion = subj.Questions[i]
|
||||
const percent = compareQuestionObj(
|
||||
currentQuestion,
|
||||
subjName,
|
||||
question,
|
||||
subj.Name
|
||||
)
|
||||
|
||||
if (percent.avg >= minMatchAmmount) {
|
||||
result.push({
|
||||
q: currentQuestion,
|
||||
match: percent.avg,
|
||||
detailedMatch: percent,
|
||||
})
|
||||
}
|
||||
|
||||
if (searchTillMatchPercent && percent.avg >= searchTillMatchPercent) {
|
||||
stopSearch = true
|
||||
}
|
||||
|
||||
i--
|
||||
}
|
||||
|
||||
result = result.sort((q1, q2) => {
|
||||
if (q1.match < q2.match) {
|
||||
return 1
|
||||
} else if (q1.match > q2.match) {
|
||||
return -1
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
})
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
export function getSubjectDifference(
|
||||
subjects: Subject[],
|
||||
subjectsToMerge: Subject[]
|
||||
): { newData: Subject[]; newSubjects: Subject[] } {
|
||||
const newData: Subject[] = []
|
||||
const newSubjects: Subject[] = []
|
||||
subjectsToMerge.forEach((remoteSubj) => {
|
||||
const localSubj = subjects.find((ls) => ls.Name === remoteSubj.Name)
|
||||
if (!localSubj) {
|
||||
newSubjects.push(remoteSubj)
|
||||
return
|
||||
}
|
||||
const addedQuestions: Question[] = []
|
||||
remoteSubj.Questions.forEach((remoteQuestion) => {
|
||||
const searchResult = searchSubject(
|
||||
localSubj,
|
||||
remoteQuestion,
|
||||
localSubj.Name,
|
||||
95 // FIXME: maybe fine tune
|
||||
)
|
||||
|
||||
if (searchResult.length === 0) {
|
||||
addedQuestions.push(remoteQuestion)
|
||||
}
|
||||
})
|
||||
if (addedQuestions.length > 0) {
|
||||
newData.push({
|
||||
Name: localSubj.Name,
|
||||
Questions: addedQuestions,
|
||||
})
|
||||
}
|
||||
})
|
||||
return { newData: newData, newSubjects: newSubjects }
|
||||
}
|
||||
|
||||
export function cleanDb(
|
||||
{
|
||||
questions: recievedQuestions,
|
||||
subjToClean,
|
||||
overwriteBeforeDate,
|
||||
qdbIndex,
|
||||
}: {
|
||||
questions: Question[]
|
||||
subjToClean: string
|
||||
overwriteBeforeDate: number
|
||||
qdbIndex: number
|
||||
},
|
||||
qdbs: QuestionDb[]
|
||||
): number[][] {
|
||||
const subjIndex = qdbs[qdbIndex].data.findIndex((x) => {
|
||||
return x.Name.toLowerCase().includes(subjToClean.toLowerCase())
|
||||
})
|
||||
|
||||
if (!qdbs[qdbIndex].data[subjIndex]) {
|
||||
return recievedQuestions.map(() => [])
|
||||
}
|
||||
|
||||
// FIXME: compare images & data too!
|
||||
const questionIndexesToRemove = recievedQuestions.map((recievedQuestion) =>
|
||||
qdbs[qdbIndex].data[subjIndex].Questions.reduce<number[]>(
|
||||
(acc, question, i) => {
|
||||
const res = compareString(
|
||||
simplifyQuestion(recievedQuestion.Q),
|
||||
simplifyQuestion(question.Q)
|
||||
)
|
||||
|
||||
if (
|
||||
res > minMatchToNotSearchOtherSubjects &&
|
||||
(!question.data.date ||
|
||||
question.data.date < overwriteBeforeDate)
|
||||
) {
|
||||
// questions indexes in subject, that should be
|
||||
// removed because of recievedQuestion
|
||||
return [...acc, i]
|
||||
}
|
||||
return acc
|
||||
},
|
||||
[]
|
||||
)
|
||||
)
|
||||
|
||||
return questionIndexesToRemove
|
||||
}
|
||||
|
||||
export function removeCacheFromQuestion(question: Question): Question {
|
||||
if (question.cache) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
const { cache, ...questionWithoutCache } = question
|
||||
return questionWithoutCache
|
||||
} else {
|
||||
return question
|
||||
}
|
||||
}
|
||||
|
||||
export function getAvailableQdbIndexes(
|
||||
qdbs: QuestionDb[],
|
||||
count = 1,
|
||||
initialIndex?: number
|
||||
): number[] {
|
||||
const indexes = qdbs.map((x) => x.index)
|
||||
const availableIndexes: number[] = []
|
||||
|
||||
const minCount = count < 1 ? 1 : count
|
||||
|
||||
let i = initialIndex || 0
|
||||
while (availableIndexes.length < minCount) {
|
||||
if (!indexes.includes(i)) {
|
||||
availableIndexes.push(i)
|
||||
}
|
||||
i += 1
|
||||
}
|
||||
return availableIndexes
|
||||
}
|
||||
|
|
@ -36,6 +36,7 @@ interface WorkerObj {
|
|||
free: Boolean
|
||||
}
|
||||
|
||||
// FIXME: type depending on type
|
||||
export interface TaskObject {
|
||||
type:
|
||||
| 'work'
|
||||
|
|
@ -44,6 +45,7 @@ export interface TaskObject {
|
|||
| 'newdb'
|
||||
| 'dbClean'
|
||||
| 'rmQuestions'
|
||||
| 'merge'
|
||||
data:
|
||||
| {
|
||||
searchIn: number[]
|
||||
|
|
@ -57,11 +59,11 @@ export interface TaskObject {
|
|||
}
|
||||
| { dbIndex: number; edits: Edits }
|
||||
| QuestionDb
|
||||
| Result
|
||||
| Omit<Result, 'qdbName'>
|
||||
| {
|
||||
questions: Question[]
|
||||
subjToClean: string
|
||||
overwriteFromDate: number
|
||||
overwriteBeforeDate: number
|
||||
qdbIndex: number
|
||||
}
|
||||
| {
|
||||
|
|
@ -70,6 +72,10 @@ export interface TaskObject {
|
|||
qdbIndex: number
|
||||
recievedQuestions: Question[]
|
||||
}
|
||||
| {
|
||||
localQdbIndex: number
|
||||
remoteQdb: QuestionDb
|
||||
}
|
||||
}
|
||||
|
||||
interface PendingJob {
|
||||
|
|
@ -90,7 +96,7 @@ interface DoneEvent extends EventEmitter {
|
|||
emit(event: 'done', res: WorkerResult): boolean
|
||||
}
|
||||
|
||||
const alertOnPendingCount = 50
|
||||
const alertOnPendingCount = 100
|
||||
const workerFile = './src/utils/classes.ts'
|
||||
let workers: Array<WorkerObj>
|
||||
let getInitData: () => Array<QuestionDb> = null
|
||||
|
|
@ -136,11 +142,10 @@ export function doALongTask(
|
|||
targetWorkerIndex?: number
|
||||
): Promise<WorkerResult> {
|
||||
if (Object.keys(pendingJobs).length > alertOnPendingCount) {
|
||||
logger.Log(
|
||||
console.error(
|
||||
`More than ${alertOnPendingCount} callers waiting for free resource! (${
|
||||
Object.keys(pendingJobs).length
|
||||
})`,
|
||||
logger.GetColor('redbg')
|
||||
})`
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue