added nearly complete p2p implementation

This commit is contained in:
mrfry 2023-03-20 18:02:45 +01:00
parent 11dacdae64
commit 5c22f575dd
25 changed files with 14320 additions and 12563 deletions

View file

@ -22,15 +22,16 @@ const recDataFile = './stats/recdata'
const dataLockFile = './data/lockData'
import logger from '../utils/logger'
import {
createQuestion,
WorkerResult,
SearchResultQuestion,
} from '../utils/classes'
import { WorkerResult } from '../utils/classes'
import { doALongTask, msgAllWorker } from './workerPool'
import idStats from '../utils/ids'
import utils from '../utils/utils'
import { addQuestion, getSubjNameWithoutYear } from './classes'
import {
addQuestion,
createQuestion,
getSubjNameWithoutYear,
SearchResultQuestion,
} from '../utils/qdbUtils'
// types
import {
@ -40,6 +41,7 @@ import {
User,
DataFile,
} from '../types/basicTypes'
import { countOfQdbs } from './qdbUtils'
// if a recievend question doesnt match at least this % to any other question in the db it gets
// added to db
@ -321,8 +323,11 @@ function runCleanWorker(
subjName: string,
qdb: QuestionDb
) {
// FIXME: clean worker should compare images too!
// see: classes.ts:1011
return
if (qdb.overwrites && qdb.overwrites.length) {
// check if subject needs to be updated, and qdb has overwriteFromDate
// check if subject needs to be updated, and qdb has overwriteBeforeDate
const overwrite = qdb.overwrites.find((x) => {
return subjName.toLowerCase().includes(x.subjName.toLowerCase())
})
@ -343,7 +348,7 @@ function runCleanWorker(
data: {
questions: recievedQuesitons,
subjToClean: subjName,
overwriteFromDate: overwrite.overwriteFromDate,
overwriteBeforeDate: overwrite.overwriteBeforeDate,
qdbIndex: qdb.index,
},
}).then(({ result: questionIndexesToRemove }) => {
@ -391,15 +396,13 @@ export function updateQuestionsInArray(
questions: Question[],
newQuestions: Question[]
): Question[] {
const indexesToRemove = questionIndexesToRemove.reduce((acc, x) => {
if (x.length > 1) {
return [...acc, ...x]
}
return acc
}, [])
if (newQuestions.length !== questionIndexesToRemove.length) {
throw new Error('newQuestions length ne questionIndexesToRemove length')
}
const indexesToRemove = questionIndexesToRemove.flat()
const newQuestionsToAdd: Question[] = newQuestions.filter((_q, i) => {
return questionIndexesToRemove[i].length > 1
return questionIndexesToRemove[i].length >= 1
})
return [
@ -407,7 +410,7 @@ export function updateQuestionsInArray(
return !indexesToRemove.includes(i)
}),
...newQuestionsToAdd.map((x) => {
x.data.date = new Date()
x.data.date = new Date().getTime()
return x
}),
]
@ -500,6 +503,10 @@ export function loadJSON(
const dataPath = dataDir + dataFile.path
if (!utils.FileExists(dataPath)) {
logger.Log(
`${dataPath} data file did not exist, created empty one!`,
'yellowbg'
)
utils.WriteFile(JSON.stringify([]), dataPath)
}
@ -520,14 +527,7 @@ export function loadJSON(
return acc
}, [])
let subjCount = 0
let questionCount = 0
res.forEach((qdb) => {
subjCount += qdb.data.length
qdb.data.forEach((subj) => {
questionCount += subj.Questions.length
})
})
const { subjCount, questionCount } = countOfQdbs(res)
logger.Log(
`Loaded ${subjCount} subjects with ${questionCount} questions from ${res.length} question db-s`,
logger.GetColor('green')
@ -543,6 +543,7 @@ export function writeData(data: Array<Subject>, path: string): void {
return {
Name: subj.Name,
Questions: subj.Questions.map((question) => {
// removing cache here
return {
Q: question.Q,
A: question.A,

File diff suppressed because it is too large Load diff

View file

@ -93,16 +93,13 @@ function DebugLog(msg: string) {
}
}
// FIXME: this might not work: what is col exactly, and how we use AddColumn?
function AddColumn(
db: Database,
table: string,
col: { [key: string]: string | number }
colName: string,
colType: string
): RunResult {
try {
const colName = Object.keys(col)[0]
const colType = col.type
const command = `ALTER TABLE ${table} ADD COLUMN ${colName} ${colType}`
const stmt = PrepareStatement(db, command)

53
src/utils/encryption.ts Normal file
View file

@ -0,0 +1,53 @@
/* ----------------------------------------------------------------------------
Question Server
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
------------------------------------------------------------------------- */
import { Crypt, RSA } from 'hybrid-crypto-js'
const rsa = new RSA()
const crypt = new Crypt()
export const createKeyPair = (): Promise<{
publicKey: string
privateKey: string
}> => {
return rsa.generateKeyPairAsync()
}
export const encrypt = (publicKey: string, text: string): string => {
return crypt.encrypt(publicKey, text)
}
export const decrypt = (privateKey: string, text: string): string => {
return crypt.decrypt(privateKey, text).message
}
export const isKeypairValid = (
publicKey: string,
privateKey: string
): boolean => {
const testText = 'nem volt jobb ötletem na'
try {
const encryptedText = encrypt(publicKey, testText)
const decryptedText = decrypt(privateKey, encryptedText)
return decryptedText === testText
} catch (e) {
return false
}
}

View file

@ -389,6 +389,9 @@ function C(color?: string): string {
if (color === 'redbg') {
return '\x1b[41m'
}
if (color === 'yellowbg') {
return '\x1b[43m\x1b[30m'
}
if (color === 'bluebg') {
return '\x1b[44m'
}
@ -416,6 +419,23 @@ function C(color?: string): string {
return '\x1b[0m'
}
function logTable(table: (string | number)[][]): void {
table.forEach((row, i) => {
const rowString: string[] = []
row.forEach((cell, j) => {
const cellColor = j === 0 || i === 0 ? 'blue' : 'green'
let cellVal = ''
if (!isNaN(+cell)) {
cellVal = cell.toLocaleString()
} else {
cellVal = cell.toString()
}
rowString.push(C(cellColor) + cellVal + C())
})
Log(rowString.join('\t'))
})
}
export default {
getColoredDateString: getColoredDateString,
Log: Log,
@ -431,4 +451,5 @@ export default {
logDir: logDir,
vlogDir: vlogDir,
setLoggingDisabled: setLoggingDisabled,
logTable: logTable,
}

606
src/utils/qdbUtils.ts Normal file
View file

@ -0,0 +1,606 @@
import logger from './logger'
import {
Question,
QuestionData,
QuestionDb,
Subject,
} from '../types/basicTypes'
interface DetailedMatch {
qMatch: number
aMatch: number
dMatch: number
matchedSubjName: string
avg: number
}
export interface SearchResultQuestion {
q: Question
match: number
detailedMatch: DetailedMatch
}
/* Percent minus for length difference */
const lengthDiffMultiplier = 10
// const commonUselessStringParts = [',', '\\.', ':', '!', '\\+', '\\s*\\.']
/* Minimum ammount to consider that two questions match during answering */
const minMatchAmmount = 75
const magicNumber = 0.7 // same as minMatchAmmount, but /100
export const minMatchToNotSearchOtherSubjects = 90
/* If all of the results are below this match percent (when only one subject is searched due to
* subject name matching) then all subjects are searched for answer */
export const noPossibleAnswerMatchPenalty = 5
const commonUselessAnswerParts = [
'A helyes válasz az ',
'A helyes válasz a ',
'A helyes válaszok: ',
'A helyes válaszok:',
'A helyes válasz: ',
'A helyes válasz:',
'The correct answer is:',
"'",
]
export function getSubjNameWithoutYear(subjName: string): string {
const t = subjName.split(' - ')
if (t[0].match(/^[0-9]{4}\/[0-9]{2}\/[0-9]{1}$/i)) {
return t[1] || subjName
} else {
return subjName
}
}
function simplifyString(toremove: string): string {
return toremove.replace(/\s/g, ' ').replace(/\s+/g, ' ').toLowerCase()
}
function removeStuff(
value: string,
removableStrings: Array<string>,
toReplace?: string
): string {
removableStrings.forEach((removableString) => {
const regex = new RegExp(removableString, 'g')
value = value.replace(regex, toReplace || '')
})
return value
}
// damn nonbreaking space
function normalizeSpaces(input: string): string {
return input.replace(/\s/g, ' ')
}
function removeUnnecesarySpaces(toremove: string): string {
return normalizeSpaces(toremove)
.replace(/\s+/g, ' ')
.replace(/(\r\n|\n|\r)/gm, '')
.trim()
}
export function compareString(
s1: string,
s2: string,
s1cache?: Array<string>,
s2cache?: Array<string>
): number {
const s1a = s1cache || s1.split(' ')
const s2a = s2cache || s2.split(' ')
if (s1 === s2) {
return 100
}
if (!s1a || !s2a) {
if (!s1a && !s2a) {
return 100
} else {
return 0
}
}
if (s1a.length < 0 || s2a.length < 0) {
if (s1a.length === 0 && s2a.length === 0) {
return 100
} else {
return 0
}
}
let match = 0
let lastMatchIndex = -2
let i = 0
while (i < s1a.length) {
if (match / i < magicNumber) {
break
}
const currMatchIndex = s2a.indexOf(s1a[i])
if (currMatchIndex !== -1 && lastMatchIndex < currMatchIndex) {
match++
lastMatchIndex = currMatchIndex
}
i++
}
let percent = Math.round(
parseFloat(((match / s1a.length) * 100).toFixed(2))
)
const lengthDifference = Math.abs(s2a.length - s1a.length)
percent -= lengthDifference * lengthDiffMultiplier
if (percent < 0) {
percent = 0
}
return percent
}
function answerPreProcessor(value: string): string {
if (!value) {
return value
}
return removeStuff(value, commonUselessAnswerParts)
}
// 'a. pécsi sör' -> 'pécsi sör'
function removeAnswerLetters(value: string): string {
if (!value) {
return value
}
const val = value.split('. ')
if (val[0].length < 2 && val.length > 1) {
val.shift()
return val.join(' ')
} else {
return value
}
}
function simplifyQA(value: string, mods: Array<Function>): string {
if (!value) {
return value
}
return mods.reduce((res, fn) => {
return fn(res)
}, value)
}
function simplifyAnswer(value: string): string {
if (!value) {
return value
}
return simplifyQA(value, [
removeUnnecesarySpaces,
answerPreProcessor,
removeAnswerLetters,
])
}
export function simplifyQuestion(question: string): string {
if (!question) {
return question
}
return simplifyQA(question, [removeUnnecesarySpaces, removeAnswerLetters])
}
function simplifyQuestionObj(question: Question): Question {
if (!question) {
return question
}
if (question.Q) {
question.Q = simplifyQA(question.Q, [
removeUnnecesarySpaces,
removeAnswerLetters,
])
}
if (question.A) {
question.A = simplifyQA(question.A, [
removeUnnecesarySpaces,
removeAnswerLetters,
])
}
return question
}
export function createQuestion(
question: Question | string,
answer?: string,
data?: QuestionData
): Question {
try {
if (typeof question === 'string') {
return {
Q: simplifyQuestion(question),
A: answer ? simplifyAnswer(answer) : undefined,
data: data,
cache: {
Q: question ? simplifyString(question).split(' ') : [],
A: answer ? simplifyString(answer).split(' ') : [],
},
}
} else {
return {
...question,
cache: {
Q: question.Q ? simplifyString(question.Q).split(' ') : [],
A: question.A ? simplifyString(question.A).split(' ') : [],
},
}
}
} catch (err) {
logger.Log('Error creating question', logger.GetColor('redbg'))
console.error(question, answer, data)
console.error(err)
return null
}
}
function compareImage(data: QuestionData, data2: QuestionData): number {
if (data.hashedImages && data2.hashedImages) {
return compareString(
data.hashedImages.join(' '),
data2.hashedImages.join(' '),
data.hashedImages,
data2.hashedImages
)
} else if (data.images && data2.images) {
return (
compareString(
data.images.join(' '),
data2.images.join(' '),
data.images,
data2.images
) - 10
)
} else {
return 0
}
}
function compareData(q1: Question, q2: Question): number {
try {
if (q1.data.type === q2.data.type) {
const dataType = q1.data.type
if (dataType === 'simple') {
return -1
} else if (dataType === 'image') {
return compareImage(q1.data, q2.data)
} else {
logger.DebugLog(
`Unhandled data type ${dataType}`,
'Compare question data',
1
)
logger.DebugLog(q1, 'Compare question data', 2)
}
} else {
return 0
}
} catch (error) {
logger.DebugLog('Error comparing data', 'Compare question data', 1)
logger.DebugLog(error.message, 'Compare question data', 1)
logger.DebugLog(error, 'Compare question data', 2)
console.error(error)
}
return 0
}
function compareQuestion(q1: Question, q2: Question): number {
return compareString(q1.Q, q2.Q, q1.cache.Q, q2.cache.Q)
// return compareString(
// q1.Q,
// q1.Q ? q1.Q.split(' ') : [],
// q2.Q,
// q2.Q ? q2.Q.split(' ') : []
// )
}
function compareAnswer(q1: Question, q2: Question): number {
return compareString(q1.A, q2.A, q1.cache.A, q2.cache.A)
// return compareString(
// q1.A,
// q1.A ? q1.A.split(' ') : [],
// q2.A,
// q2.A ? q2.A.split(' ') : []
// )
}
export function compareQuestionObj(
q1: Question,
_q1subjName: string,
q2: Question,
q2subjName: string
): DetailedMatch {
const qMatch = compareQuestion(q1, q2)
const aMatch = q2.A ? compareAnswer(q1, q2) : 0
// -1 if botth questions are simple
const dMatch = compareData(q1, q2)
let avg = -1
if (q2.A) {
if (dMatch === -1) {
avg = Math.min(qMatch, aMatch)
} else {
avg = Math.min(qMatch, aMatch, dMatch)
}
} else {
if (dMatch === -1) {
avg = qMatch
} else {
avg = Math.min(qMatch, dMatch)
}
}
return {
qMatch: qMatch,
aMatch: aMatch,
dMatch: dMatch,
matchedSubjName: q2subjName,
avg: avg,
}
}
function questionToString(question: Question): string {
const { Q, A, data } = question
if (data.type !== 'simple') {
return '?' + Q + '\n!' + A + '\n>' + JSON.stringify(data)
} else {
return '?' + Q + '\n!' + A
}
}
function subjectToString(subj: Subject): string {
const { Questions, Name } = subj
const result: string[] = []
Questions.forEach((question) => {
result.push(questionToString(question))
})
return '+' + Name + '\n' + result.join('\n')
}
export function addQuestion(
data: Array<Subject>,
subj: string,
question: Question
): void {
logger.DebugLog('Adding new question with subjName: ' + subj, 'qdb add', 1)
logger.DebugLog(question, 'qdb add', 3)
const i = data.findIndex((subject) => {
return (
subject.Name &&
subj
.toLowerCase()
.includes(getSubjNameWithoutYear(subject.Name).toLowerCase())
)
})
if (i !== -1) {
logger.DebugLog('Adding new question to existing subject', 'qdb add', 1)
data[i].Questions.push(question)
} else {
logger.Log(`Creating new subject: "${subj}"`)
data.push({
Name: subj,
Questions: [question],
})
}
}
export function prepareQuestion(question: Question): Question {
return simplifyQuestionObj(createQuestion(question))
}
export function dataToString(data: Array<Subject>): string {
const result: string[] = []
data.forEach((subj) => {
result.push(subjectToString(subj))
})
return result.join('\n\n')
}
export function countQuestionsInSubject(subject: Subject): number {
return subject.Questions.length
}
export function countQuestionsInSubjects(subject: Subject[]): number {
let questionCount = 0
subject.forEach((subj) => {
questionCount += countQuestionsInSubject(subj)
})
return questionCount
}
export function countOfQdb(qdb: QuestionDb): {
subjCount: number
questionCount: number
} {
const subjCount = qdb.data.length
const questionCount = countQuestionsInSubjects(qdb.data)
return { subjCount: subjCount, questionCount: questionCount }
}
export function countOfQdbs(qdbs: QuestionDb[]): {
subjCount: number
questionCount: number
} {
let questionCount = 0
let subjCount = 0
qdbs.forEach((qdb) => {
const { subjCount: sc, questionCount: qc } = countOfQdb(qdb)
questionCount += qc
subjCount += sc
})
return { subjCount: subjCount, questionCount: questionCount }
}
export function searchSubject(
subj: Subject,
question: Question,
subjName: string,
searchTillMatchPercent?: number
): SearchResultQuestion[] {
let result: SearchResultQuestion[] = []
let stopSearch = false
let i = subj.Questions.length - 1
while (i >= 0 && !stopSearch) {
const currentQuestion = subj.Questions[i]
const percent = compareQuestionObj(
currentQuestion,
subjName,
question,
subj.Name
)
if (percent.avg >= minMatchAmmount) {
result.push({
q: currentQuestion,
match: percent.avg,
detailedMatch: percent,
})
}
if (searchTillMatchPercent && percent.avg >= searchTillMatchPercent) {
stopSearch = true
}
i--
}
result = result.sort((q1, q2) => {
if (q1.match < q2.match) {
return 1
} else if (q1.match > q2.match) {
return -1
} else {
return 0
}
})
return result
}
export function getSubjectDifference(
subjects: Subject[],
subjectsToMerge: Subject[]
): { newData: Subject[]; newSubjects: Subject[] } {
const newData: Subject[] = []
const newSubjects: Subject[] = []
subjectsToMerge.forEach((remoteSubj) => {
const localSubj = subjects.find((ls) => ls.Name === remoteSubj.Name)
if (!localSubj) {
newSubjects.push(remoteSubj)
return
}
const addedQuestions: Question[] = []
remoteSubj.Questions.forEach((remoteQuestion) => {
const searchResult = searchSubject(
localSubj,
remoteQuestion,
localSubj.Name,
95 // FIXME: maybe fine tune
)
if (searchResult.length === 0) {
addedQuestions.push(remoteQuestion)
}
})
if (addedQuestions.length > 0) {
newData.push({
Name: localSubj.Name,
Questions: addedQuestions,
})
}
})
return { newData: newData, newSubjects: newSubjects }
}
export function cleanDb(
{
questions: recievedQuestions,
subjToClean,
overwriteBeforeDate,
qdbIndex,
}: {
questions: Question[]
subjToClean: string
overwriteBeforeDate: number
qdbIndex: number
},
qdbs: QuestionDb[]
): number[][] {
const subjIndex = qdbs[qdbIndex].data.findIndex((x) => {
return x.Name.toLowerCase().includes(subjToClean.toLowerCase())
})
if (!qdbs[qdbIndex].data[subjIndex]) {
return recievedQuestions.map(() => [])
}
// FIXME: compare images & data too!
const questionIndexesToRemove = recievedQuestions.map((recievedQuestion) =>
qdbs[qdbIndex].data[subjIndex].Questions.reduce<number[]>(
(acc, question, i) => {
const res = compareString(
simplifyQuestion(recievedQuestion.Q),
simplifyQuestion(question.Q)
)
if (
res > minMatchToNotSearchOtherSubjects &&
(!question.data.date ||
question.data.date < overwriteBeforeDate)
) {
// questions indexes in subject, that should be
// removed because of recievedQuestion
return [...acc, i]
}
return acc
},
[]
)
)
return questionIndexesToRemove
}
export function removeCacheFromQuestion(question: Question): Question {
if (question.cache) {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { cache, ...questionWithoutCache } = question
return questionWithoutCache
} else {
return question
}
}
export function getAvailableQdbIndexes(
qdbs: QuestionDb[],
count = 1,
initialIndex?: number
): number[] {
const indexes = qdbs.map((x) => x.index)
const availableIndexes: number[] = []
const minCount = count < 1 ? 1 : count
let i = initialIndex || 0
while (availableIndexes.length < minCount) {
if (!indexes.includes(i)) {
availableIndexes.push(i)
}
i += 1
}
return availableIndexes
}

View file

@ -36,6 +36,7 @@ interface WorkerObj {
free: Boolean
}
// FIXME: type depending on type
export interface TaskObject {
type:
| 'work'
@ -44,6 +45,7 @@ export interface TaskObject {
| 'newdb'
| 'dbClean'
| 'rmQuestions'
| 'merge'
data:
| {
searchIn: number[]
@ -57,11 +59,11 @@ export interface TaskObject {
}
| { dbIndex: number; edits: Edits }
| QuestionDb
| Result
| Omit<Result, 'qdbName'>
| {
questions: Question[]
subjToClean: string
overwriteFromDate: number
overwriteBeforeDate: number
qdbIndex: number
}
| {
@ -70,6 +72,10 @@ export interface TaskObject {
qdbIndex: number
recievedQuestions: Question[]
}
| {
localQdbIndex: number
remoteQdb: QuestionDb
}
}
interface PendingJob {
@ -90,7 +96,7 @@ interface DoneEvent extends EventEmitter {
emit(event: 'done', res: WorkerResult): boolean
}
const alertOnPendingCount = 50
const alertOnPendingCount = 100
const workerFile = './src/utils/classes.ts'
let workers: Array<WorkerObj>
let getInitData: () => Array<QuestionDb> = null
@ -136,11 +142,10 @@ export function doALongTask(
targetWorkerIndex?: number
): Promise<WorkerResult> {
if (Object.keys(pendingJobs).length > alertOnPendingCount) {
logger.Log(
console.error(
`More than ${alertOnPendingCount} callers waiting for free resource! (${
Object.keys(pendingJobs).length
})`,
logger.GetColor('redbg')
})`
)
}