mirror of
https://gitlab.com/MrFry/mrfrys-node-server
synced 2025-04-01 20:24:18 +02:00
Merge branch 'master' of gitlab.com:MrFry/mrfrys-node-server
This commit is contained in:
commit
f44eba05d8
11 changed files with 548 additions and 128 deletions
|
@ -146,39 +146,42 @@ function setup(data: SubmoduleData): void {
|
|||
chatMessageRead({ sender: chatPartner, reciever: userid })
|
||||
})
|
||||
|
||||
socket.on('chat message', (message) => {
|
||||
const { reciever, msg, type } = message
|
||||
if (!reciever || !msg || !type) {
|
||||
return
|
||||
}
|
||||
const recieverUser = dbtools.Select(userDB, 'users', {
|
||||
id: reciever,
|
||||
})[0]
|
||||
if (!recieverUser) {
|
||||
socket.emit('chat message', {
|
||||
success: false,
|
||||
date: new Date().getTime(),
|
||||
sender: reciever,
|
||||
reciever: userid,
|
||||
type: 'text',
|
||||
msg: `A #${reciever} számú felhasználó nem létezik`,
|
||||
})
|
||||
return
|
||||
}
|
||||
socket.on(
|
||||
'chat message',
|
||||
(message: { reciever: string; msg: string; type: string }) => {
|
||||
const { reciever, msg, type } = message
|
||||
if (!reciever || !msg || !msg.trim() || !type) {
|
||||
return
|
||||
}
|
||||
const recieverUser = dbtools.Select(userDB, 'users', {
|
||||
id: reciever,
|
||||
})[0]
|
||||
if (!recieverUser) {
|
||||
socket.emit('chat message', {
|
||||
success: false,
|
||||
date: new Date().getTime(),
|
||||
sender: reciever,
|
||||
reciever: userid,
|
||||
type: 'text',
|
||||
msg: `A #${reciever} számú felhasználó nem létezik`,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
const msgObj = {
|
||||
sender: userid,
|
||||
reciever: parseInt(reciever),
|
||||
msg: dbtools.sanitizeQuery(msg),
|
||||
type: type || 'text',
|
||||
date: new Date().getTime(),
|
||||
unread: 1,
|
||||
const msgObj = {
|
||||
sender: userid,
|
||||
reciever: parseInt(reciever),
|
||||
msg: dbtools.sanitizeQuery(msg),
|
||||
type: type || 'text',
|
||||
date: new Date().getTime(),
|
||||
unread: 1,
|
||||
}
|
||||
dbtools.Insert(msgDB, 'msgs', msgObj)
|
||||
if (userid !== parseInt(reciever)) {
|
||||
io.sockets.in(reciever.toString()).emit('chat message', msgObj)
|
||||
}
|
||||
}
|
||||
dbtools.Insert(msgDB, 'msgs', msgObj)
|
||||
if (userid !== reciever) {
|
||||
io.sockets.in(reciever.toString()).emit('chat message', msgObj)
|
||||
}
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
// socket.on('disconnect', () => {})
|
||||
|
@ -201,6 +204,59 @@ function setup(data: SubmoduleData): void {
|
|||
})
|
||||
})
|
||||
|
||||
app.post('/postfeedbackfile', function (req: Request, res) {
|
||||
logger.LogReq(req)
|
||||
const user: User = req.session.user
|
||||
|
||||
utils
|
||||
.uploadFile(req, uloadFiles)
|
||||
.then(({ filePath }) => {
|
||||
const fileName = filePath.replace(publicDir, '')
|
||||
const isImage = ['png', 'jpg', 'jpeg', 'gif'].some((ext) => {
|
||||
return fileName.toLowerCase().includes(ext)
|
||||
})
|
||||
const msgObj = {
|
||||
sender: user.id,
|
||||
reciever: 1,
|
||||
msg: fileName,
|
||||
type: isImage ? 'img' : 'file',
|
||||
date: new Date().getTime(),
|
||||
unread: 1,
|
||||
}
|
||||
dbtools.Insert(msgDB, 'msgs', msgObj)
|
||||
|
||||
res.json({ success: true })
|
||||
io.sockets.in('1').emit('chat message', msgObj)
|
||||
})
|
||||
.catch(() => {
|
||||
res.json({ success: false, msg: 'error during uploading' })
|
||||
return
|
||||
})
|
||||
})
|
||||
|
||||
app.post('/postfeedback', function (req: Request<{ content: string }>, res) {
|
||||
logger.LogReq(req)
|
||||
const user: User = req.session.user
|
||||
const { content } = req.body
|
||||
if (!content || !content.trim()) {
|
||||
res.json({ success: false })
|
||||
return
|
||||
}
|
||||
|
||||
const msgObj = {
|
||||
sender: user.id,
|
||||
reciever: 1,
|
||||
msg: dbtools.sanitizeQuery(req.body.content),
|
||||
type: 'text',
|
||||
date: new Date().getTime(),
|
||||
unread: 1,
|
||||
}
|
||||
dbtools.Insert(msgDB, 'msgs', msgObj)
|
||||
|
||||
res.json({ success: true })
|
||||
io.sockets.in('1').emit('chat message', msgObj)
|
||||
})
|
||||
|
||||
app.get('/hasNewMsg', (req: Request, res) => {
|
||||
const user: User = req.session.user
|
||||
const userid: number = user.id
|
||||
|
|
|
@ -20,63 +20,14 @@
|
|||
|
||||
import { Response } from 'express'
|
||||
|
||||
import logger from '../../../utils/logger'
|
||||
import utils from '../../../utils/utils'
|
||||
import { Request, SubmoduleData, User } from '../../../types/basicTypes'
|
||||
import { Request, SubmoduleData } from '../../../types/basicTypes'
|
||||
|
||||
const msgFile = 'stats/msgs'
|
||||
const uloadFiles = 'data/f'
|
||||
|
||||
function setup(data: SubmoduleData): void {
|
||||
const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data
|
||||
|
||||
app.post('/postfeedbackfile', function (req: Request, res: Response) {
|
||||
utils
|
||||
.uploadFile(req, uloadFiles)
|
||||
.then(() => {
|
||||
res.json({ success: true })
|
||||
})
|
||||
.catch(() => {
|
||||
res.json({ success: false, msg: 'error during uploading' })
|
||||
return
|
||||
})
|
||||
|
||||
logger.LogReq(req)
|
||||
logger.Log('New feedback file', logger.GetColor('bluebg'))
|
||||
})
|
||||
|
||||
app.post('/postfeedback', function (req: Request, res: Response) {
|
||||
logger.LogReq(req)
|
||||
if (req.body.fromLogin) {
|
||||
logger.Log(
|
||||
'New feedback message from Login page',
|
||||
logger.GetColor('bluebg')
|
||||
)
|
||||
} else {
|
||||
logger.Log(
|
||||
'New feedback message from feedback page',
|
||||
logger.GetColor('bluebg')
|
||||
)
|
||||
}
|
||||
|
||||
const user: User = req.session.user
|
||||
|
||||
utils.AppendToFile(
|
||||
utils.GetDateString() +
|
||||
':\n' +
|
||||
JSON.stringify(
|
||||
{
|
||||
...req.body,
|
||||
userID: user ? user.id : 'no user',
|
||||
},
|
||||
null,
|
||||
2
|
||||
),
|
||||
msgFile
|
||||
)
|
||||
res.json({ success: true })
|
||||
})
|
||||
|
||||
app.route('/fosuploader').post(function (req: Request, res: Response) {
|
||||
utils.uploadFile(req, uloadFiles).then(({ fileName }) => {
|
||||
res.redirect('/f/' + fileName)
|
||||
|
|
|
@ -53,6 +53,7 @@ import {
|
|||
dataToString,
|
||||
getSubjNameWithoutYear,
|
||||
WorkerResult,
|
||||
SearchResultQuestion,
|
||||
// compareQuestionObj,
|
||||
} from '../../../utils/classes'
|
||||
import {
|
||||
|
@ -194,7 +195,7 @@ function searchInDbs(
|
|||
logger.DebugLog(taskResult, 'ask', 2)
|
||||
resolve({
|
||||
question: question,
|
||||
result: taskResult.result,
|
||||
result: taskResult.result as SearchResultQuestion[],
|
||||
success: true,
|
||||
})
|
||||
} catch (err) {
|
||||
|
|
|
@ -248,6 +248,7 @@ function setup(data: SubmoduleData): Submodule {
|
|||
logger.LogReq(req)
|
||||
const sessionID = req.cookies.sessionID
|
||||
const user: User = req.session.user
|
||||
const { all } = req.query
|
||||
|
||||
if (!user) {
|
||||
res.json({
|
||||
|
@ -262,10 +263,16 @@ function setup(data: SubmoduleData): Submodule {
|
|||
logger.GetColor('cyan')
|
||||
)
|
||||
|
||||
// removing session from db
|
||||
dbtools.Delete(userDB, 'sessions', {
|
||||
id: sessionID,
|
||||
})
|
||||
if (all) {
|
||||
dbtools.Delete(userDB, 'sessions', {
|
||||
userID: user.id,
|
||||
})
|
||||
} else {
|
||||
dbtools.Delete(userDB, 'sessions', {
|
||||
id: sessionID,
|
||||
})
|
||||
}
|
||||
|
||||
res.clearCookie('sessionID').json({
|
||||
msg: 'Successfull logout',
|
||||
result: 'success',
|
||||
|
|
197
src/tests/oldQuestionRemoving.test.ts
Normal file
197
src/tests/oldQuestionRemoving.test.ts
Normal file
|
@ -0,0 +1,197 @@
|
|||
import { updateQuestionsInArray } from '../utils/actions'
|
||||
import { createQuestion } from '../utils/classes'
|
||||
import { cleanDb } from '../utils/classes'
|
||||
import { QuestionDb, Subject, Question } from '../types/basicTypes'
|
||||
|
||||
const date = (x?: number) => new Date().getTime() + (x || 0)
|
||||
|
||||
const q1 = createQuestion(
|
||||
'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE',
|
||||
'piaci áruk eltérhet a névértéktől.',
|
||||
{
|
||||
type: 'simple',
|
||||
date: date(-1000),
|
||||
}
|
||||
)
|
||||
const q2 = createQuestion(
|
||||
'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE',
|
||||
'afjléa gféda gfdjs légf',
|
||||
{
|
||||
type: 'simple',
|
||||
date: date(-1000),
|
||||
}
|
||||
)
|
||||
const q3 = createQuestion(
|
||||
'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE',
|
||||
'afjlsd gfds dgfs gf sdgf d',
|
||||
{
|
||||
type: 'simple',
|
||||
date: date(-1000),
|
||||
}
|
||||
)
|
||||
const q4 = createQuestion(
|
||||
'A kötvény névértéke',
|
||||
'A kötvényen feltüntetett meghatározott nagyságú összeg.',
|
||||
{
|
||||
type: 'simple',
|
||||
date: date(-1000),
|
||||
}
|
||||
)
|
||||
const q5 = createQuestion(
|
||||
'Mi az osztalék? asd asd',
|
||||
'A vállalati profit egy része..',
|
||||
{
|
||||
type: 'simple',
|
||||
date: date(1000),
|
||||
}
|
||||
)
|
||||
const q6 = createQuestion(
|
||||
'valaim nagyon értelmes kérdés asd asd',
|
||||
'A vállalati profit egy része..',
|
||||
{
|
||||
type: 'simple',
|
||||
date: date(1000),
|
||||
}
|
||||
)
|
||||
|
||||
function setupTest({
|
||||
newQuestions,
|
||||
data,
|
||||
subjToClean,
|
||||
}: {
|
||||
newQuestions: Question[]
|
||||
data: Subject[]
|
||||
subjToClean?: string
|
||||
}) {
|
||||
const recievedQuestions: Question[] = newQuestions.map((x) => {
|
||||
return {
|
||||
...x,
|
||||
data: {
|
||||
...x.data,
|
||||
date: date(),
|
||||
},
|
||||
}
|
||||
})
|
||||
const subjName = subjToClean || 'subject'
|
||||
const overwriteFromDate = date(-100)
|
||||
const qdbIndex = 0
|
||||
const qdbs: QuestionDb[] = [
|
||||
{
|
||||
name: 'test',
|
||||
data: data,
|
||||
index: 0,
|
||||
path: '',
|
||||
shouldSearch: 'asd',
|
||||
shouldSave: {},
|
||||
},
|
||||
]
|
||||
const subjIndex = qdbs[qdbIndex].data.findIndex((x) => {
|
||||
return x.Name.toLowerCase().includes(subjName.toLowerCase())
|
||||
})
|
||||
|
||||
const questionIndexesToRemove = cleanDb(
|
||||
{
|
||||
questions: recievedQuestions,
|
||||
subjToClean: subjName,
|
||||
overwriteFromDate: overwriteFromDate,
|
||||
qdbIndex: qdbIndex,
|
||||
},
|
||||
qdbs
|
||||
)
|
||||
|
||||
const updatedQuestions = updateQuestionsInArray(
|
||||
questionIndexesToRemove,
|
||||
qdbs[qdbIndex].data[subjIndex].Questions,
|
||||
recievedQuestions
|
||||
)
|
||||
|
||||
return {
|
||||
questionIndexesToRemove: questionIndexesToRemove,
|
||||
updatedQuestions: updatedQuestions,
|
||||
overwriteFromDate: overwriteFromDate,
|
||||
subjIndex: subjIndex,
|
||||
}
|
||||
}
|
||||
|
||||
const s1: Subject = { Name: 'test subject', Questions: [q1, q2, q4, q5] }
|
||||
|
||||
test('Old and duplicate questions should be removed from the database', () => {
|
||||
const { questionIndexesToRemove, updatedQuestions, overwriteFromDate } =
|
||||
setupTest({ newQuestions: [q1, q4, q5], data: [s1] })
|
||||
|
||||
expect(questionIndexesToRemove.length).toBe(3)
|
||||
expect(questionIndexesToRemove[0].length).toBe(2)
|
||||
|
||||
expect(updatedQuestions.length).toBe(3)
|
||||
const toremoveCount = updatedQuestions.filter((question) => {
|
||||
return question.Q.includes('TOREMOVE')
|
||||
}).length
|
||||
expect(toremoveCount).toBe(1)
|
||||
const newQuestion = updatedQuestions.find((question) => {
|
||||
return question.Q.includes('TOREMOVE')
|
||||
})
|
||||
expect(newQuestion.data.date > overwriteFromDate).toBeTruthy()
|
||||
})
|
||||
|
||||
const s2: Subject = {
|
||||
Name: 'test subject',
|
||||
Questions: [q1, q2, q3, q4, q5, q6],
|
||||
}
|
||||
|
||||
test('Old and duplicate questions should be removed from the database round 2', () => {
|
||||
const { questionIndexesToRemove, updatedQuestions, overwriteFromDate } =
|
||||
setupTest({ newQuestions: [q1, q4, q5], data: [s2] })
|
||||
|
||||
expect(questionIndexesToRemove.length).toBe(3)
|
||||
expect(questionIndexesToRemove[0].length).toBe(3)
|
||||
|
||||
expect(updatedQuestions.length).toBe(4)
|
||||
const toremoveCount = updatedQuestions.filter((question) => {
|
||||
return question.Q.includes('TOREMOVE')
|
||||
}).length
|
||||
expect(toremoveCount).toBe(1)
|
||||
const newQuestion = updatedQuestions.find((question) => {
|
||||
return question.Q.includes('TOREMOVE')
|
||||
})
|
||||
expect(newQuestion.data.date > overwriteFromDate).toBeTruthy()
|
||||
})
|
||||
|
||||
const s3: Subject = {
|
||||
Name: 'test subject',
|
||||
Questions: [q5, q6].map((x) => ({
|
||||
...x,
|
||||
data: {
|
||||
...x.data,
|
||||
date: date(+50000),
|
||||
},
|
||||
})),
|
||||
}
|
||||
|
||||
test('Old and duplicate questions should be removed from the database: questions should be left alone when they are newer', () => {
|
||||
const { questionIndexesToRemove, updatedQuestions } = setupTest({
|
||||
newQuestions: [q5, q6],
|
||||
data: [s3],
|
||||
})
|
||||
|
||||
expect(questionIndexesToRemove.length).toBe(2)
|
||||
questionIndexesToRemove.forEach((x) => {
|
||||
expect(x.length).toBe(0)
|
||||
})
|
||||
|
||||
expect(updatedQuestions.length).toBe(2)
|
||||
})
|
||||
|
||||
const s4: Subject = {
|
||||
Name: 'something else',
|
||||
Questions: [q5, q6],
|
||||
}
|
||||
|
||||
test('Old and duplicate questions should be removed from the database:other subjects should be left alone', () => {
|
||||
const { subjIndex } = setupTest({
|
||||
newQuestions: [q5, q6],
|
||||
data: [s2, s1, s4, s3],
|
||||
subjToClean: 'else',
|
||||
})
|
||||
|
||||
expect(subjIndex).toBe(2)
|
||||
})
|
|
@ -56,6 +56,10 @@ export interface DataFile {
|
|||
path: string
|
||||
name: string
|
||||
locked?: Boolean
|
||||
overwrites?: Array<{
|
||||
subjName: string
|
||||
overwriteFromDate: number
|
||||
}>
|
||||
shouldSearch:
|
||||
| string
|
||||
| {
|
||||
|
|
|
@ -27,7 +27,7 @@ import {
|
|||
WorkerResult,
|
||||
SearchResultQuestion,
|
||||
} from '../utils/classes'
|
||||
import { doALongTask } from './workerPool'
|
||||
import { doALongTask, msgAllWorker } from './workerPool'
|
||||
import idStats from '../utils/ids'
|
||||
import utils from '../utils/utils'
|
||||
import { addQuestion, getSubjNameWithoutYear } from './classes'
|
||||
|
@ -201,32 +201,20 @@ function processIncomingRequestUsingDb(
|
|||
.then((results: Array<WorkerResult>) => {
|
||||
const allQuestions: Question[] = [] // all new questions here that do not have result
|
||||
results.forEach((result: WorkerResult, i) => {
|
||||
const add = result.result.every((res: SearchResultQuestion) => {
|
||||
return res.match < minMatchAmmountToAdd
|
||||
})
|
||||
const add = (result.result as SearchResultQuestion[]).every(
|
||||
(res: SearchResultQuestion) => {
|
||||
return res.match < minMatchAmmountToAdd
|
||||
}
|
||||
)
|
||||
if (add && !result.error) {
|
||||
allQuestions.push(recievedQuestions[i])
|
||||
}
|
||||
})
|
||||
|
||||
try {
|
||||
const subjName = getSubjNameWithoutYear(recievedData.subj)
|
||||
if (allQuestions.length > 0) {
|
||||
allQuestions.forEach((currentQuestion) => {
|
||||
const sName = getSubjNameWithoutYear(recievedData.subj)
|
||||
logger.DebugLog(
|
||||
'Adding question with subjName: ' + sName + ' :',
|
||||
'isadding',
|
||||
3
|
||||
)
|
||||
logger.DebugLog(currentQuestion, 'isadding', 3)
|
||||
addQuestion(qdb.data, sName, {
|
||||
...currentQuestion,
|
||||
data: {
|
||||
...currentQuestion.data,
|
||||
date: new Date().getTime(),
|
||||
},
|
||||
})
|
||||
})
|
||||
addQuestionsToDb(allQuestions, subjName, qdb)
|
||||
|
||||
currWrites++
|
||||
logger.DebugLog(
|
||||
|
@ -258,6 +246,7 @@ function processIncomingRequestUsingDb(
|
|||
qdbIndex: qdb.index,
|
||||
qdbName: qdb.name,
|
||||
})
|
||||
runCleanWorker(recievedData.quiz, subjName, qdb)
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
logger.Log(
|
||||
|
@ -287,6 +276,124 @@ function processIncomingRequestUsingDb(
|
|||
})
|
||||
}
|
||||
|
||||
function addQuestionsToDb(
|
||||
allQuestions: Question[],
|
||||
subjName: string,
|
||||
qdb: QuestionDb
|
||||
) {
|
||||
allQuestions.forEach((currentQuestion) => {
|
||||
logger.DebugLog(
|
||||
'Adding question with subjName: ' + subjName + ' :',
|
||||
'isadding',
|
||||
3
|
||||
)
|
||||
logger.DebugLog(currentQuestion, 'isadding', 3)
|
||||
addQuestion(qdb.data, subjName, {
|
||||
...currentQuestion,
|
||||
data: {
|
||||
...currentQuestion.data,
|
||||
date: new Date().getTime(),
|
||||
},
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function runCleanWorker(
|
||||
recievedQuesitons: Question[],
|
||||
subjName: string,
|
||||
qdb: QuestionDb
|
||||
) {
|
||||
if (qdb.overwrites && qdb.overwrites.length) {
|
||||
// check if subject needs to be updated, and qdb has overwriteFromDate
|
||||
const overwrite = qdb.overwrites.find((x) => {
|
||||
return subjName.toLowerCase().includes(x.subjName.toLowerCase())
|
||||
})
|
||||
|
||||
if (!overwrite) {
|
||||
return
|
||||
}
|
||||
// logger.Log(
|
||||
// `\tStarting cleaning in subject "${logger.C(
|
||||
// 'green'
|
||||
// )}${subjName}${logger.C('')}" (matched: "${logger.C('green')}${
|
||||
// overwrite.subjName
|
||||
// }${logger.C('')}")`
|
||||
// )
|
||||
// pass recieved questions to a worker
|
||||
doALongTask({
|
||||
type: 'dbClean',
|
||||
data: {
|
||||
questions: recievedQuesitons,
|
||||
subjToClean: subjName,
|
||||
overwriteFromDate: overwrite.overwriteFromDate,
|
||||
qdbIndex: qdb.index,
|
||||
},
|
||||
}).then(({ result: questionIndexesToRemove }) => {
|
||||
const subjIndex = qdb.data.findIndex((x) => {
|
||||
return x.Name.toLowerCase().includes(subjName.toLowerCase())
|
||||
})
|
||||
// sends msgs to all workers to remove it too
|
||||
|
||||
msgAllWorker({
|
||||
type: 'rmQuestions',
|
||||
data: {
|
||||
questionIndexesToRemove: questionIndexesToRemove as number[][],
|
||||
subjIndex: subjIndex,
|
||||
qdbIndex: qdb.index,
|
||||
recievedQuestions: recievedQuesitons,
|
||||
},
|
||||
})
|
||||
|
||||
// it adds the recieved question WITH DATE!
|
||||
// recievedQuestions doesnt have date-s
|
||||
qdb.data[subjIndex].Questions = updateQuestionsInArray(
|
||||
questionIndexesToRemove as number[][],
|
||||
qdb.data[subjIndex].Questions,
|
||||
recievedQuesitons
|
||||
)
|
||||
|
||||
// saves the file
|
||||
writeData(qdb.data, qdb.path)
|
||||
logger.Log(
|
||||
`\tRemoved ${logger.C('green')}${
|
||||
(questionIndexesToRemove as number[][]).filter(
|
||||
(x: number[]) => x.length > 1
|
||||
).length
|
||||
}${logger.C()} old questions from ${logger.C(
|
||||
'green'
|
||||
)}${subjName}${logger.C()}`
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export function updateQuestionsInArray(
|
||||
questionIndexesToRemove: number[][],
|
||||
questions: Question[],
|
||||
newQuestions: Question[]
|
||||
): Question[] {
|
||||
const indexesToRemove = questionIndexesToRemove.reduce((acc, x) => {
|
||||
if (x.length > 1) {
|
||||
return [...acc, ...x]
|
||||
}
|
||||
return acc
|
||||
}, [])
|
||||
|
||||
const newQuestionsToAdd: Question[] = newQuestions.filter((_q, i) => {
|
||||
return questionIndexesToRemove[i].length > 1
|
||||
})
|
||||
|
||||
return [
|
||||
...questions.filter((_x, i) => {
|
||||
return !indexesToRemove.includes(i)
|
||||
}),
|
||||
...newQuestionsToAdd.map((x) => {
|
||||
x.data.date = new Date()
|
||||
return x
|
||||
}),
|
||||
]
|
||||
}
|
||||
|
||||
export function isQuestionValid(question: Question): boolean {
|
||||
if (!question.Q) {
|
||||
return false
|
||||
|
@ -435,12 +542,12 @@ export function backupData(questionDbs: Array<QuestionDb>): void {
|
|||
const path = './publicDirs/qminingPublic/backs/'
|
||||
utils.CreatePath(path)
|
||||
try {
|
||||
logger.Log(`Backing up ${data.name}...`)
|
||||
// logger.Log(`Backing up ${data.name}...`)
|
||||
writeData(
|
||||
data.data,
|
||||
`${path}${data.name}_${utils.GetDateString(undefined, true)}.json`
|
||||
)
|
||||
logger.Log('Done')
|
||||
// logger.Log('Done')
|
||||
} catch (err) {
|
||||
logger.Log(
|
||||
`Error backing up data file ${data.name}!`,
|
||||
|
|
|
@ -17,6 +17,7 @@
|
|||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
------------------------------------------------------------------------- */
|
||||
// FIXME: this should be renamed to worker.ts or something
|
||||
|
||||
import { isMainThread, parentPort, workerData } from 'worker_threads'
|
||||
|
||||
|
@ -27,12 +28,13 @@ import {
|
|||
QuestionDb,
|
||||
Subject,
|
||||
} from '../types/basicTypes'
|
||||
import { editDb, Edits } from './actions'
|
||||
import { editDb, Edits, updateQuestionsInArray } from './actions'
|
||||
// import { TaskObject } from './workerPool'
|
||||
|
||||
export interface WorkerResult {
|
||||
msg: string
|
||||
workerIndex: number
|
||||
result?: SearchResultQuestion[]
|
||||
result?: SearchResultQuestion[] | number[][]
|
||||
error?: boolean
|
||||
}
|
||||
|
||||
|
@ -115,12 +117,15 @@ function removeUnnecesarySpaces(toremove: string): string {
|
|||
return normalizeSpaces(toremove).replace(/\s+/g, ' ')
|
||||
}
|
||||
|
||||
export function compareString(
|
||||
function compareString(
|
||||
s1: string,
|
||||
s1a: Array<string>,
|
||||
s2: string,
|
||||
s2a: Array<string>
|
||||
s1cache?: Array<string>,
|
||||
s2cache?: Array<string>
|
||||
): number {
|
||||
const s1a = s1cache || s1.split(' ')
|
||||
const s2a = s2cache || s2.split(' ')
|
||||
|
||||
if (s1 === s2) {
|
||||
return 100
|
||||
}
|
||||
|
@ -277,16 +282,16 @@ function compareImage(data: QuestionData, data2: QuestionData): number {
|
|||
if (data.hashedImages && data2.hashedImages) {
|
||||
return compareString(
|
||||
data.hashedImages.join(' '),
|
||||
data.hashedImages,
|
||||
data2.hashedImages.join(' '),
|
||||
data.hashedImages,
|
||||
data2.hashedImages
|
||||
)
|
||||
} else if (data.images && data2.images) {
|
||||
return (
|
||||
compareString(
|
||||
data.images.join(' '),
|
||||
data.images,
|
||||
data2.images.join(' '),
|
||||
data.images,
|
||||
data2.images
|
||||
) - 10
|
||||
)
|
||||
|
@ -324,7 +329,7 @@ function compareData(q1: Question, q2: Question): number {
|
|||
}
|
||||
|
||||
function compareQuestion(q1: Question, q2: Question): number {
|
||||
return compareString(q1.Q, q1.cache.Q, q2.Q, q2.cache.Q)
|
||||
return compareString(q1.Q, q2.Q, q1.cache.Q, q2.cache.Q)
|
||||
// return compareString(
|
||||
// q1.Q,
|
||||
// q1.Q ? q1.Q.split(' ') : [],
|
||||
|
@ -334,7 +339,7 @@ function compareQuestion(q1: Question, q2: Question): number {
|
|||
}
|
||||
|
||||
function compareAnswer(q1: Question, q2: Question): number {
|
||||
return compareString(q1.A, q1.cache.A, q2.A, q2.cache.A)
|
||||
return compareString(q1.A, q2.A, q1.cache.A, q2.cache.A)
|
||||
// return compareString(
|
||||
// q1.A,
|
||||
// q1.A ? q1.A.split(' ') : [],
|
||||
|
@ -648,7 +653,7 @@ if (!isMainThread) {
|
|||
`[THREAD #${workerIndex}]: Worker ${workerIndex} reporting for duty`
|
||||
)
|
||||
|
||||
parentPort.on('message', (msg) => {
|
||||
parentPort.on('message', (msg /*: TaskObject */) => {
|
||||
if (msg.type === 'work') {
|
||||
const {
|
||||
subjName,
|
||||
|
@ -814,15 +819,93 @@ if (!isMainThread) {
|
|||
workerIndex: workerIndex,
|
||||
})
|
||||
// console.log(`[THREAD #${workerIndex}]: newdb`)
|
||||
} else if (msg.type === 'dbClean') {
|
||||
const removedIndexes = cleanDb(msg.data, qdbs)
|
||||
|
||||
const workerResult: WorkerResult = {
|
||||
msg: `From thread #${workerIndex}: db clean done`,
|
||||
workerIndex: workerIndex,
|
||||
result: removedIndexes,
|
||||
}
|
||||
|
||||
parentPort.postMessage(workerResult)
|
||||
} else if (msg.type === 'rmQuestions') {
|
||||
const {
|
||||
questionIndexesToRemove,
|
||||
subjIndex,
|
||||
qdbIndex,
|
||||
recievedQuestions,
|
||||
} = msg.data
|
||||
|
||||
qdbs[qdbIndex].data[subjIndex].Questions = updateQuestionsInArray(
|
||||
questionIndexesToRemove,
|
||||
qdbs[qdbIndex].data[subjIndex].Questions,
|
||||
recievedQuestions
|
||||
)
|
||||
|
||||
parentPort.postMessage({
|
||||
msg: `From thread #${workerIndex}: rm question done`,
|
||||
workerIndex: workerIndex,
|
||||
})
|
||||
} else {
|
||||
logger.Log(`Invalid msg type!`, logger.GetColor('redbg'))
|
||||
console.error(msg)
|
||||
|
||||
parentPort.postMessage({
|
||||
msg: `From thread #${workerIndex}: Invalid message type (${msg.type})!`,
|
||||
workerIndex: workerIndex,
|
||||
})
|
||||
}
|
||||
})
|
||||
} else {
|
||||
// console.log('[THREAD]: Main thread!')
|
||||
}
|
||||
|
||||
export function cleanDb(
|
||||
{
|
||||
questions: recievedQuestions,
|
||||
subjToClean,
|
||||
overwriteFromDate,
|
||||
qdbIndex,
|
||||
}: {
|
||||
questions: Question[]
|
||||
subjToClean: string
|
||||
overwriteFromDate: number
|
||||
qdbIndex: number
|
||||
},
|
||||
qdbs: QuestionDb[]
|
||||
): number[][] {
|
||||
const subjIndex = qdbs[qdbIndex].data.findIndex((x) => {
|
||||
return x.Name.toLowerCase().includes(subjToClean.toLowerCase())
|
||||
})
|
||||
|
||||
if (!qdbs[qdbIndex].data[subjIndex]) {
|
||||
return recievedQuestions.map(() => [])
|
||||
}
|
||||
|
||||
const questionIndexesToRemove = recievedQuestions.map((recievedQuestion) =>
|
||||
qdbs[qdbIndex].data[subjIndex].Questions.reduce<number[]>(
|
||||
(acc, question, i) => {
|
||||
const res = compareString(
|
||||
simplifyQuestion(recievedQuestion.Q),
|
||||
simplifyQuestion(question.Q)
|
||||
)
|
||||
|
||||
if (
|
||||
res > minMatchToNotSearchOtherSubjects &&
|
||||
(!question.data.date || question.data.date < overwriteFromDate)
|
||||
) {
|
||||
return [...acc, i]
|
||||
}
|
||||
return acc
|
||||
},
|
||||
[]
|
||||
)
|
||||
)
|
||||
|
||||
return questionIndexesToRemove
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
|
|
|
@ -41,11 +41,6 @@ const logFileName = 'log'
|
|||
const writeInterval = 10
|
||||
const debugLevel = parseInt(process.env.NS_LOGLEVEL) || 0
|
||||
|
||||
// let vdata = {} // visit data
|
||||
// let dvData = {} // visit data, but daily
|
||||
// let uData = {} // visit data, but per user
|
||||
// let udvData = {} // visit data, but per user and daily
|
||||
|
||||
let vData = {} // visit data
|
||||
let dvData = {} // visit data, but daily
|
||||
let uvData = {} // visit data, but per user
|
||||
|
@ -237,12 +232,19 @@ function LogStat(url: string, hostname: string, userId: number | string): void {
|
|||
|
||||
url = hostname + url.split('?')[0]
|
||||
Inc(url)
|
||||
AddUserIdStat(userId.toString())
|
||||
IncUserStat(userId.toString())
|
||||
AddVisitStat(url)
|
||||
if (shouldAddUserStat(url)) {
|
||||
AddUserIdStat(userId.toString())
|
||||
IncUserStat(userId.toString())
|
||||
}
|
||||
Save()
|
||||
}
|
||||
|
||||
const userStatExcludes = ['stable.user.js', 'infos', 'hasNewMsg']
|
||||
function shouldAddUserStat(url: string) {
|
||||
return !userStatExcludes.some((x) => url.includes(x))
|
||||
}
|
||||
|
||||
function IncUserStat(userId: string) {
|
||||
try {
|
||||
if (uvData[userId] === undefined) {
|
||||
|
|
|
@ -34,8 +34,8 @@ interface WorkerObj {
|
|||
free: Boolean
|
||||
}
|
||||
|
||||
interface TaskObject {
|
||||
type: 'work' | 'dbEdit' | 'newQuestions' | 'newdb'
|
||||
export interface TaskObject {
|
||||
type: 'work' | 'dbEdit' | 'newQuestions' | 'newdb' | 'dbClean' | 'rmQuestions'
|
||||
data:
|
||||
| {
|
||||
searchIn: number[]
|
||||
|
@ -50,6 +50,18 @@ interface TaskObject {
|
|||
| { dbIndex: number; edits: Edits }
|
||||
| QuestionDb
|
||||
| Result
|
||||
| {
|
||||
questions: Question[]
|
||||
subjToClean: string
|
||||
overwriteFromDate: number
|
||||
qdbIndex: number
|
||||
}
|
||||
| {
|
||||
questionIndexesToRemove: number[][]
|
||||
subjIndex: number
|
||||
qdbIndex: number
|
||||
recievedQuestions: Question[]
|
||||
}
|
||||
}
|
||||
|
||||
interface PendingJob {
|
||||
|
|
|
@ -1 +1 @@
|
|||
Subproject commit ce63911b68e5604c02f5761dd4b5388272b76c6f
|
||||
Subproject commit 9c333008c7049b032e4e3d0dbf5fd5893542ffb7
|
Loading…
Add table
Add a link
Reference in a new issue