feat: complete rewrite for gigabit-scale performance and SQLite storage

.env.example

@@ -1,4 +1,2 @@
 NICK=
-# pass= from Set-Cookie header in lower security mode
-# set-cookie: pass=<...>; ...
 PASS=

.github/workflows/go.yml

@@ -16,7 +16,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v4
       with:
-        go-version: '1.23.2'
+        go-version: '1.26.1'
 
     - name: Build
       run: go build -v ./...

.gitignore

@@ -1 +1,3 @@
 .env.local
+*.db
+*.exe

go.mod

@@ -1,15 +1,24 @@
 module ncore-leaderboard
 
-go 1.23.0
+go 1.26.1
-
-toolchain go1.24.0
 
 require (
-	github.com/PuerkitoBio/goquery v1.10.2
 	github.com/joho/godotenv v1.5.1
+	github.com/valyala/fasthttp v1.69.0
+	modernc.org/sqlite v1.47.0
 )
 
 require (
-	github.com/andybalholm/cascadia v1.3.3 // indirect
+	github.com/andybalholm/brotli v1.2.0 // indirect
-	golang.org/x/net v0.37.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/klauspost/compress v1.18.5 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/ncruces/go-strftime v1.0.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	golang.org/x/sys v0.42.0 // indirect
+	modernc.org/libc v1.70.0 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.11.0 // indirect
 )

go.sum

@@ -1,73 +1,63 @@
-github.com/PuerkitoBio/goquery v1.10.2 h1:7fh2BdHcG6VFZsK7toXBT/Bh1z5Wmy8Q9MV9HqT2AM8=
+github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
-github.com/PuerkitoBio/goquery v1.10.2/go.mod h1:0guWGjcLu9AYC7C1GHnpysHy056u9aEkUHwhdnePMCU=
+github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
-github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
-github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
+github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
-github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
-golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
-golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
-golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
-golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
-golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+github.com/valyala/fasthttp v1.69.0 h1:fNLLESD2SooWeh2cidsuFtOcrEi4uB4m1mPrkJMZyVI=
-golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+github.com/valyala/fasthttp v1.69.0/go.mod h1:4wA4PfAraPlAsJ5jMSqCE2ug5tqUPwKXxVj8oNECGcw=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
-golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
-golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
-golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
-golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
+golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
-golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
+golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c=
+golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
-golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
+golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+modernc.org/ccgo/v4 v4.32.0 h1:hjG66bI/kqIPX1b2yT6fr/jt+QedtP2fqojG2VrFuVw=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+modernc.org/ccgo/v4 v4.32.0/go.mod h1:6F08EBCx5uQc38kMGl+0Nm0oWczoo1c7cgpzEry7Uc0=
-golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
-golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
-golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+modernc.org/libc v1.70.0 h1:U58NawXqXbgpZ/dcdS9kMshu08aiA6b7gusEusqzNkw=
-golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+modernc.org/libc v1.70.0/go.mod h1:OVmxFGP1CI/Z4L3E0Q3Mf1PDE0BucwMkcXjjLntvHJo=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
-golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
-golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
+modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
-golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
-golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+modernc.org/sqlite v1.47.0 h1:R1XyaNpoW4Et9yly+I2EeX7pBza/w+pmYee/0HJDyKk=
-golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
+modernc.org/sqlite v1.47.0/go.mod h1:hWjRO6Tj/5Ik8ieqxQybiEOUXy0NJFNp2tpvVpKlvig=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
-golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
-golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
-golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

justfile

@@ -0,0 +1,34 @@
+set shell := ["powershell", "-NoProfile", "-Command"]
+
+# Build the most optimized executable for modern CPUs
+build:
+    $env:GOAMD64="v3"; go build -ldflags="-s -w" -trimpath -o ncore-leaderboard.exe main.go
+    @echo "Build complete: ncore-leaderboard.exe"
+
+# Run the scraper
+run: build
+    ./ncore-leaderboard.exe
+
+# Cleanup all database and build artifacts
+clean:
+    if (Test-Path ncore-leaderboard.exe) { Remove-Item ncore-leaderboard.exe -ErrorAction SilentlyContinue }
+    if (Test-Path leaderboard.db) { Remove-Item leaderboard.db -ErrorAction SilentlyContinue }
+    if (Test-Path leaderboard.db-wal) { Remove-Item leaderboard.db-wal -ErrorAction SilentlyContinue }
+    if (Test-Path leaderboard.db-shm) { Remove-Item leaderboard.db-shm -ErrorAction SilentlyContinue }
+    @echo "Cleanup complete."
+
+# Query the database (Example: just query 100)
+query rank="1":
+    sqlite3 leaderboard.db "SELECT * FROM leaderboard WHERE rank = {{ rank }} LIMIT 1;"
+
+# Show top N from the database (Example: just top 5)
+top n="10":
+    sqlite3 leaderboard.db "SELECT * FROM leaderboard ORDER BY rank ASC LIMIT {{ n }};"
+
+# Formatting
+fmt:
+    go fmt ./...
+
+# Tidy dependencies
+tidy:
+    go mod tidy

main.go

@@ -1,192 +1,296 @@
 package main
 
 import (
-	"encoding/csv"
+	"bytes"
+	"cmp"
+	"context"
+	"database/sql"
 	"fmt"
 	"log"
-	"net/http"
 	"os"
-	"strconv"
+	"os/signal"
-	"strings"
+	"runtime"
+	"slices"
 	"sync"
 	"sync/atomic"
 	"time"
 
-	"github.com/PuerkitoBio/goquery"
 	"github.com/joho/godotenv"
+	"github.com/valyala/fasthttp"
+	_ "modernc.org/sqlite"
 )
 
 const (
 	baseURL      = "https://ncore.pro/profile.php?id="
 	startProfile = 1
 	endProfile   = 1812000
-	concurrency  = 50
+	concurrency  = 1000
-	outputFile   = "output.log"
+	dbFile       = "leaderboard.db"
-	writeBatch   = 100
+
+	maxConns     = 2000
+	readTimeout  = 10 * time.Second
+	writeTimeout = 5 * time.Second
+	maxRetries   = 2
 )
 
+type Profile struct {
+	ID   uint32
+	Rank uint32
+}
+
 var (
 	nick      string
 	pass      string
-	client    *http.Client
+	processed atomic.Int64
-	wg        sync.WaitGroup
+	found     atomic.Int64
-	mu        sync.Mutex
+	errors    atomic.Int64
-	lines     []Line
+	authFail  atomic.Bool
-	processed int32
+	client    *fasthttp.Client
 )
 
-type Line struct {
-	URL       string
-	SecondCol int
-}
-
 func init() {
 	_ = godotenv.Load(".env.local")
-	godotenv.Load()
+	_ = godotenv.Load()
 	nick = os.Getenv("NICK")
 	pass = os.Getenv("PASS")
-	client = &http.Client{}
+
+	client = &fasthttp.Client{
+		Name:                     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
+		MaxConnsPerHost:          maxConns,
+		MaxIdleConnDuration:      90 * time.Second,
+		ReadTimeout:              readTimeout,
+		WriteTimeout:             writeTimeout,
+		NoDefaultUserAgentHeader: false,
+		Dial: (&fasthttp.TCPDialer{
+			Concurrency:      concurrency,
+			DNSCacheDuration: 1 * time.Hour,
+		}).Dial,
+	}
 }
 
-func fetchProfile(id int) {
+func fastParseRank(data []byte) (uint32, bool) {
+	labelMarker := []byte("<div class=\"profil_jobb_elso2\">Helyezés:</div>")
+	idx := bytes.Index(data, labelMarker)
+	if idx == -1 {
+		return 0, false
+	}
+
+	searchArea := data[idx:]
+	valueMarker := []byte("profil_jobb_masodik2\">")
+	vIdx := bytes.Index(searchArea, valueMarker)
+	if vIdx == -1 {
+		return 0, false
+	}
+
+	start := vIdx + len(valueMarker)
+	end := bytes.IndexByte(searchArea[start:], '<')
+	if end == -1 {
+		return 0, false
+	}
+
+	raw := searchArea[start : start+end]
+	var val uint32
+	hasDigits := false
+	for _, b := range raw {
+		if b >= '0' && b <= '9' {
+			val = val*10 + uint32(b-'0')
+			hasDigits = true
+		}
+	}
+
+	return val, hasDigits
+}
+
+func worker(ctx context.Context, jobs <-chan uint32, results chan<- Profile, wg *sync.WaitGroup) {
 	defer wg.Done()
-	url := fmt.Sprintf("%s%d", baseURL, id)
+	cookie := fmt.Sprintf("nick=%s; pass=%s", nick, pass)
 
-	req, err := http.NewRequest("GET", url, nil)
+	for id := range jobs {
-	if err != nil {
+		select {
-		log.Printf("Error creating request for %d: %v\n", id, err)
+		case <-ctx.Done():
 			return
-	}
+		default:
-	req.Header.Set("Cookie", fmt.Sprintf("nick=%s; pass=%s", nick, pass))
+			processed.Add(1)
+			var body []byte
+			success := false
 
-	resp, err := client.Do(req)
+			for r := 0; r <= maxRetries; r++ {
-	if err != nil {
+				req := fasthttp.AcquireRequest()
-		log.Printf("Error fetching profile %d: %v\n", id, err)
+				resp := fasthttp.AcquireResponse()
-		return
+				req.SetRequestURI(fmt.Sprintf("%s%d", baseURL, id))
-	}
+				req.Header.Set("Cookie", cookie)
-	defer resp.Body.Close()
+				req.Header.Set("Accept-Encoding", "gzip")
 
-	if resp.StatusCode != http.StatusOK {
+				if err := client.Do(req, resp); err == nil && resp.StatusCode() == 200 {
-		return
+					encoding := resp.Header.Peek("Content-Encoding")
+					if bytes.Equal(encoding, []byte("gzip")) {
+						body, _ = resp.BodyGunzip()
+					} else {
+						body = make([]byte, len(resp.Body()))
+						copy(body, resp.Body())
 					}
 
-	doc, err := goquery.NewDocumentFromReader(resp.Body)
+					if bytes.Contains(body, []byte("Belépés")) {
-	if err != nil {
+						authFail.Store(true)
-		log.Printf("Error parsing profile document for %d: %v\n", id, err)
+						fasthttp.ReleaseRequest(req)
-		return
+						fasthttp.ReleaseResponse(resp)
+						break
 					}
 
-	doc.Find(".userbox_tartalom_mini").Each(func(i int, s *goquery.Selection) {
+					if rank, ok := fastParseRank(body); ok {
-		s.Find(".profil_jobb_elso2").Each(func(ii int, labelSel *goquery.Selection) {
+						results <- Profile{ID: id, Rank: rank}
-			label := labelSel.Text()
+						found.Add(1)
-			valueSel := labelSel.Next()
-			if valueSel.Length() > 0 {
-				value := valueSel.Text()
-				switch label {
-				case "Helyezés:":
-					rank := strings.TrimSuffix(value, ".")
-					rankInt, err := strconv.Atoi(rank)
-					if err != nil {
-						log.Printf("Skipping profile %d due to invalid rank: %s\n", id, rank)
-						return
 					}
-					mu.Lock()
+					success = true
-					lines = append(lines, Line{URL: url, SecondCol: rankInt})
+					fasthttp.ReleaseRequest(req)
-					mu.Unlock()
+					fasthttp.ReleaseResponse(resp)
-					atomic.AddInt32(&processed, 1)
+					break
-					if atomic.LoadInt32(&processed)%writeBatch == 0 {
-						writeSortedOutput()
 				}
-					printProgress()
+				fasthttp.ReleaseRequest(req)
+				fasthttp.ReleaseResponse(resp)
+				if r < maxRetries {
+					time.Sleep(time.Duration(r+1) * 100 * time.Millisecond)
 				}
 			}
-		})
+			if !success {
-	})
+				errors.Add(1)
-}
-
-func printProgress() {
-	fmt.Printf("\rProcessed %d profiles...", atomic.LoadInt32(&processed))
-}
-
-func quicksort(lines []Line, low, high int) {
-	if low < high {
-		p := partition(lines, low, high)
-		quicksort(lines, low, p-1)
-		quicksort(lines, p+1, high)
 			}
-}
-
-func partition(lines []Line, low, high int) int {
-	pivot := lines[high].SecondCol
-	i := low - 1
-	for j := low; j < high; j++ {
-		if lines[j].SecondCol < pivot {
-			i++
-			lines[i], lines[j] = lines[j], lines[i]
-		}
-	}
-	lines[i+1], lines[high] = lines[high], lines[i+1]
-	return i + 1
-}
-
-func sortLinesQuick() {
-	if len(lines) > 1 {
-		quicksort(lines, 0, len(lines)-1)
-	}
-}
-
-func writeSortedOutput() {
-	mu.Lock()
-	defer mu.Unlock()
-
-	sortLinesQuick()
-
-	file, err := os.Create(outputFile)
-	if err != nil {
-		log.Fatalf("Error creating output file: %v\n", err)
-	}
-	defer file.Close()
-
-	writer := csv.NewWriter(file)
-	defer writer.Flush()
-
-	for _, line := range lines {
-		if err := writer.Write([]string{line.URL, strconv.Itoa(line.SecondCol)}); err != nil {
-			log.Printf("Error writing line to output file: %v\n", err)
 		}
 	}
 }
 
 func main() {
-	if _, err := os.Stat(outputFile); err == nil {
+	runtime.GOMAXPROCS(runtime.NumCPU())
-		var response string
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
-		fmt.Printf("Output file %s already exists. Overwrite? (yes/no): ", outputFile)
+	defer cancel()
-		fmt.Scanln(&response)
+
-		if response != "yes" {
+	if nick == "" || pass == "" {
-			log.Println("Exiting. Please rename or remove the existing output file.")
+		log.Fatal("NICK and PASS environment variables are required.")
-			return
-		}
-		err := os.Remove(outputFile)
-		if err != nil {
-			log.Fatalf("Failed to remove existing output file: %v\n", err)
-		}
 	}
 
-	fmt.Println("Scraping in progress...")
+	total := endProfile - startProfile + 1
-	startTime := time.Now()
+	fmt.Printf("Leaderboard: Workers=%d, Cores=%d, Range=[%d-%d]\n", concurrency, runtime.NumCPU(), startProfile, endProfile)
 
-	for i := startProfile; i <= endProfile; i++ {
+	jobs := make(chan uint32, concurrency*10)
+	resultsChan := make(chan Profile, concurrency*10)
+	var wg sync.WaitGroup
+
+	results := make([]Profile, 0, 500000)
+	collectorWg := sync.WaitGroup{}
+	collectorWg.Add(1)
+	go func() {
+		defer collectorWg.Done()
+		for p := range resultsChan {
+			results = append(results, p)
+		}
+	}()
+
+	for i := 0; i < concurrency; i++ {
 		wg.Add(1)
-		go fetchProfile(i)
+		go func() {
+			worker(ctx, jobs, resultsChan, &wg)
+		}()
+	}
 
-		if i%concurrency == 0 {
+	startTime := time.Now()
-			wg.Wait()
+	go func() {
+		ticker := time.NewTicker(1 * time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				p := processed.Load()
+				f := found.Load()
+				e := errors.Load()
+				if authFail.Load() {
+					fmt.Print("\nWARNING: Auth failure/Rate limit detected!\n")
+					authFail.Store(false)
+				}
+				if p == 0 {
+					continue
+				}
+				elapsed := time.Since(startTime).Seconds()
+				rps := float64(p) / elapsed
+				eta := time.Duration(float64(int64(total)-p)/rps) * time.Second
+				fmt.Printf("\rProgress: %d/%d (%.2f%%) | Found: %d | Err: %d | Speed: %.0f/s | ETA: %v    ",
+					p, total, float64(p)/float64(total)*100, f, e, rps, eta.Round(time.Second))
 			}
 		}
+	}()
+
+	for i := uint32(startProfile); i <= uint32(endProfile); i++ {
+		select {
+		case jobs <- i:
+		case <-ctx.Done():
+			goto Done
+		}
+	}
+
+Done:
+	close(jobs)
 	wg.Wait()
+	close(resultsChan)
+	collectorWg.Wait()
 
-	writeSortedOutput()
+	fmt.Printf("\nScraping complete. Finalizing %d results...\n", len(results))
 
-	elapsedTime := time.Since(startTime)
+	slices.SortFunc(results, func(a, b Profile) int {
-	fmt.Printf("\nScraping and sorting completed in %s\n", elapsedTime)
+		return cmp.Compare(a.Rank, b.Rank)
+	})
+
+	if _, err := os.Stat(dbFile); err == nil {
+		os.Remove(dbFile)
+	}
+
+	db, err := sql.Open("sqlite", dbFile)
+	if err != nil {
+		log.Fatalf("Failed to open SQLite: %v", err)
+	}
+	defer db.Close()
+
+	db.Exec("PRAGMA journal_mode = WAL")
+	db.Exec("PRAGMA synchronous = OFF")
+	db.Exec("PRAGMA cache_size = 1000000")
+	db.Exec("CREATE TABLE leaderboard (rank INTEGER, id INTEGER)")
+
+	fmt.Printf("Writing to %s...\n", dbFile)
+	tx, err := db.Begin()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	stmt, err := tx.Prepare("INSERT INTO leaderboard(rank, id) VALUES(?, ?)")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer stmt.Close()
+
+	for _, p := range results {
+		_, err = stmt.Exec(p.Rank, p.ID)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	fmt.Println("Committing transaction...")
+	if err := tx.Commit(); err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println("Indexing ranks...")
+	db.Exec("CREATE INDEX idx_rank ON leaderboard(rank)")
+	db.Exec("CREATE INDEX idx_id ON leaderboard(id)")
+
+	fmt.Println("\n--- TOP 5 LEADERBOARD ---")
+	limit := 5
+	if len(results) < 5 {
+		limit = len(results)
+	}
+	for i := 0; i < limit; i++ {
+		fmt.Printf("#%-8d: ID %-8d (%s%d)\n", results[i].Rank, results[i].ID, baseURL, results[i].ID)
+	}
+
+	fmt.Printf("\nSuccess! Database saved in %v\n", time.Since(startTime).Round(time.Second))
 }