From 5084c7029544f156bd20123e727760480473d28c Mon Sep 17 00:00:00 2001
From: skidoodle <contact@albert.lol>
Date: Thu, 12 Jun 2025 22:42:22 +0200
Subject: [PATCH] Refactor database handling and API endpoints

- Removed fsnotify dependency and related file watching logic.
- Introduced SQLite database for storing user profiles and history.
- Created new API endpoints for fetching latest profiles and user history.
- Migrated existing JSON data to the new database structure.
- Simplified HTML and JavaScript for fetching and displaying profile data.
- Improved error handling and logging throughout the application.
---
 go.mod             |   3 +-
 go.sum             |   5 +-
 index.html         | 347 +++++++++----------------------------
 main.go            | 416 +++++++++++++++++++++++----------------------
 migrate/migrate.go | 174 +++++++++++++++++++
 5 files changed, 471 insertions(+), 474 deletions(-)
 create mode 100644 migrate/migrate.go

diff --git a/go.mod b/go.mod
index bf9a240..401d17f 100644
--- a/go.mod
+++ b/go.mod
@@ -6,12 +6,11 @@ toolchain go1.23.2
 
 require (
 	github.com/PuerkitoBio/goquery v1.10.1
-	github.com/fsnotify/fsnotify v1.8.0
 	github.com/joho/godotenv v1.5.1
+	github.com/mattn/go-sqlite3 v1.14.28
 )
 
 require (
 	github.com/andybalholm/cascadia v1.3.3 // indirect
 	golang.org/x/net v0.33.0 // indirect
-	golang.org/x/sys v0.28.0 // indirect
 )
diff --git a/go.sum b/go.sum
index 72f047f..b25b048 100644
--- a/go.sum
+++ b/go.sum
@@ -2,11 +2,11 @@ github.com/PuerkitoBio/goquery v1.10.1 h1:Y8JGYUkXWTGRB6Ars3+j3kN0xg1YqqlwvdTV8W
 github.com/PuerkitoBio/goquery v1.10.1/go.mod h1:IYiHrOMps66ag56LEH7QYDDupKXyo5A8qrjIx3ZtujY=
 github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
 github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
-github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
-github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A=
+github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
@@ -46,7 +46,6 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
 golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
diff --git a/index.html b/index.html
index 6e711e5..580d6f8 100644
--- a/index.html
+++ b/index.html
@@ -8,313 +8,122 @@
     <script src="//unpkg.com/boxicons"></script>
     <script src="//unpkg.com/chart.js"></script>
     <style>
-      body {
-        background-color: #121212;
-        color: #efefef;
-        font-family: 'Roboto', sans-serif;
-      }
-
-      #historyModal {
-        transition: opacity 0.3s ease-in-out;
-      }
-
-      #historyModal.hidden {
-        opacity: 0;
-        pointer-events: none;
-      }
-
-      #historyModal.visible {
-        opacity: 1;
-      }
-
-      #profiles {
-        display: flex;
-        flex-wrap: wrap;
-        gap: 1rem;
-        justify-content: center;
-      }
-
-      @media (min-width: 768px) {
-        #profiles {
-          grid-template-columns: repeat(2, minmax(0, 1fr));
-        }
-      }
-
-      @media (min-width: 1024px) {
-        #profiles {
-          grid-template-columns: repeat(3, minmax(0, 1fr));
-        }
-      }
-
-      .profile-card {
-        flex: 0 1 calc(100% - 2rem);
-        max-width: 400px;
-        background-color: #1f1f1f;
-        color: #e5e5e5;
-      }
-
-      .profile-card button {
-        background-color: #3b3b3b;
-        color: #ffffff;
-      }
-
-      .profile-card button:hover {
-        background-color: #5b5b5b;
-      }
-
-      #historyModal .bg-gray-800 {
-        background-color: #2c2c2c;
-        color: #ffffff;
-        overflow-y: auto;
-      }
-
-      #historyModal button {
-        background-color: #444444;
-      }
-
-      #historyModal button:hover {
-        background-color: #666666;
-      }
-
-      .button-container {
-        display: flex;
-        flex-wrap: wrap;
-        gap: 1rem;
-      }
-
-      @media (max-width: 640px) {
-        #historyModal .bg-gray-800 {
-          padding: 1.5rem;
-          width: 90%;
-        }
-      }
-
-      #historyChart {
-        height: 300px;
-      }
+      body { background-color: #121212; color: #efefef; font-family: 'Roboto', sans-serif; }
+      #historyModal { transition: opacity 0.3s ease-in-out; }
+      #historyModal.hidden { opacity: 0; pointer-events: none; }
+      #historyModal.visible { opacity: 1; }
+      #profiles { display: flex; flex-wrap: wrap; gap: 1rem; justify-content: center; }
+      .profile-card { flex: 0 1 calc(100% - 2rem); max-width: 400px; background-color: #1f1f1f; color: #e5e5e5; }
+      .profile-card button { background-color: #3b3b3b; color: #ffffff; }
+      .profile-card button:hover { background-color: #5b5b5b; }
+      #historyModal .bg-gray-800 { background-color: #2c2c2c; color: #ffffff; overflow-y: auto; }
+      #historyModal button { background-color: #444444; }
+      #historyModal button:hover { background-color: #666666; }
     </style>
   </head>
   <body class="p-8 flex flex-col items-center">
-    <h1 class="text-4xl mb-10 font-semibold text-gray-100 text-center">
-      nCore Profile Stats
-    </h1>
-
+    <h1 class="text-4xl mb-10 font-semibold text-gray-100 text-center">nCore Profile Stats</h1>
     <div id="profiles" class="w-full max-w-7xl"></div>
-
-    <div
-      id="historyModal"
-      class="fixed inset-0 hidden bg-black bg-opacity-50 flex items-center justify-center z-50 mt-6"
-      aria-hidden="true"
-      aria-modal="true"
-      tabindex="-1">
-      <div
-        class="bg-gray-800 rounded-lg p-8 w-11/12 max-w-3xl text-white shadow-lg relative">
+    <div id="historyModal" class="fixed inset-0 hidden bg-black bg-opacity-50 flex items-center justify-center z-50 mt-6">
+      <div class="bg-gray-800 rounded-lg p-8 w-11/12 max-w-3xl text-white shadow-lg relative">
         <h2 class="text-2xl mb-6 font-semibold" id="modal-profile-name"></h2>
         <canvas id="historyChart" height="250"></canvas>
-        <button
-          class="mt-4 px-6 py-2 bg-red-600 hover:bg-red-700 text-white rounded"
-          onclick="closeModal()">
-          Close
-        </button>
-        <button
-          class="absolute top-2 right-2 text-white"
-          onclick="closeModal()">
-          <i class="bx bx-x text-2xl"></i>
-        </button>
+        <button class="mt-4 px-6 py-2 bg-red-600 hover:bg-red-700 text-white rounded" onclick="closeModal()">Close</button>
+        <button class="absolute top-2 right-2 text-white" onclick="closeModal()"><i class="bx bx-x text-2xl"></i></button>
       </div>
     </div>
 
     <script>
-      let allProfileData = []
-      let historyChart
+      let historyChart;
 
+      // Fetches only the LATEST data for each profile for a fast initial load.
       async function fetchProfiles() {
-        const response = await fetch('/data')
-        allProfileData = await response.json()
+        const response = await fetch('/api/profiles');
+        const latestRecords = await response.json();
 
-        const profilesDiv = document.getElementById('profiles')
-        profilesDiv.innerHTML = ''
+        const profilesDiv = document.getElementById('profiles');
+        profilesDiv.innerHTML = '';
 
-        const profileGroups = groupByOwner(allProfileData)
-
-        for (const [owner, records] of Object.entries(profileGroups)) {
-          const latestRecord = records[records.length - 1]
-          const profileCard = document.createElement('div')
-          profileCard.classList.add(
-            'profile-card',
-            'rounded-lg',
-            'shadow-lg',
-            'p-6',
-            'w-full',
-            'flex',
-            'flex-col',
-            'justify-between'
-          )
+        for (const latestRecord of latestRecords) {
+          const profileCard = document.createElement('div');
+          profileCard.classList.add('profile-card', 'rounded-lg', 'shadow-lg', 'p-6', 'w-full', 'flex', 'flex-col', 'justify-between');
 
           profileCard.innerHTML = `
-            <h2 class="text-2xl font-semibold mb-4"><i class='bx bxs-user text-2xl mr-2'></i>${owner}</h2>
-            <div class="space-y-2">
-                <p><i class='bx bx-trophy mr-2'></i> Rank: ${latestRecord.rank}</p>
-                <p><i class='bx bx-cloud-upload mr-2'></i> Upload: ${latestRecord.upload}</p>
-                <p><i class='bx bx-upload mr-2'></i> Current Upload: ${latestRecord.current_upload}</p>
-                <p><i class='bx bx-download mr-2'></i> Current Download: ${latestRecord.current_download}</p>
-                <p><i class='bx bx-coin mr-2'></i> Points: ${latestRecord.points}</p>
-                <p><i class='bx bx-folder mr-2'></i> Seeding Count: ${latestRecord.seeding_count}</p>
+            <div>
+              <h2 class="text-2xl font-semibold mb-4"><i class='bx bxs-user text-2xl mr-2'></i>${latestRecord.owner}</h2>
+              <div class="space-y-2">
+                  <p><i class='bx bx-trophy mr-2'></i> Rank: ${latestRecord.rank}</p>
+                  <p><i class='bx bx-cloud-upload mr-2'></i> Upload: ${latestRecord.upload}</p>
+                  <p><i class='bx bx-upload mr-2'></i> Current Upload: ${latestRecord.current_upload}</p>
+                  <p><i class='bx bx-download mr-2'></i> Current Download: ${latestRecord.current_download}</p>
+                  <p><i class='bx bx-coin mr-2'></i> Points: ${latestRecord.points}</p>
+                  <p><i class='bx bx-folder mr-2'></i> Seeding Count: ${latestRecord.seeding_count}</p>
+              </div>
             </div>
-            <button class="mt-6 px-4 py-2 text-white rounded" onclick="showHistory('${owner}', event)">View History</button>
-          `
-
-          profilesDiv.appendChild(profileCard)
+            <button class="mt-6 px-4 py-2 text-white rounded" onclick="showHistory('${latestRecord.owner}', event)">View History</button>
+          `;
+          profilesDiv.appendChild(profileCard);
         }
       }
 
-      function groupByOwner(data) {
-        return data.reduce((acc, record) => {
-          acc[record.owner] = acc[record.owner] || []
-          acc[record.owner].push(record)
-          return acc
-        }, {})
-      }
+      // Fetches and displays the history for a SINGLE profile ON DEMAND.
+      async function showHistory(owner, event) {
+        event.stopPropagation();
+        const historyModal = document.getElementById('historyModal');
+        document.getElementById('modal-profile-name').innerText = `History for ${owner}`;
 
-      function parseStorageValue(value) {
-        const numericValue = parseFloat(value)
-        if (value.includes('TiB')) {
-          return numericValue.toFixed(2)
-        } else if (value.includes('GiB')) {
-          return (numericValue / 1024).toFixed(2)
-        } else {
-          return numericValue.toFixed(2)
+        // Fetch history for this specific user from the new API endpoint
+        const response = await fetch(`/api/history?owner=${encodeURIComponent(owner)}`);
+        const profileHistory = await response.json();
+
+        if (!profileHistory || profileHistory.length === 0) {
+            alert('No history data found for this user.');
+            return;
         }
-      }
 
-      function parseSpeedValue(value) {
-        if (value.includes('KiB')) {
-          return (parseFloat(value) / 1024).toFixed(2)
-        } else if (value.includes('MiB')) {
-          return parseFloat(value).toFixed(2)
-        } else {
-          return parseFloat(value).toFixed(2)
-        }
-      }
-
-      function showHistory(owner, event) {
-        event.stopPropagation()
-        const historyModal = document.getElementById('historyModal')
-        document.getElementById('modal-profile-name').innerText = owner
-
-        const profileHistory = allProfileData.filter(
-          record => record.owner === owner
-        )
-
-        const labels = profileHistory.map(record =>
-          new Date(record.timestamp).toLocaleDateString()
-        )
-        const rankData = profileHistory.map(record => record.rank)
-        const uploadData = profileHistory.map(record =>
-          parseStorageValue(record.upload)
-        )
-        const currentUploadData = profileHistory.map(record =>
-          parseSpeedValue(record.current_upload)
-        )
-        const downloadData = profileHistory.map(record =>
-          parseSpeedValue(record.current_download)
-        )
-        const pointsData = profileHistory.map(record => record.points)
-
-        const seedingCount = profileHistory.map(record => record.seeding_count)
+        const labels = profileHistory.map(record => new Date(record.timestamp).toLocaleDateString());
+        const rankData = profileHistory.map(record => record.rank);
+        const uploadData = profileHistory.map(record => parseStorageValue(record.upload));
+        const pointsData = profileHistory.map(record => record.points);
+        const seedingCount = profileHistory.map(record => record.seeding_count);
 
         const chartData = {
           labels: labels,
           datasets: [
-            {
-              label: 'Rank',
-              data: rankData,
-              borderColor: 'rgba(255, 99, 132, 1)',
-              fill: false,
-            },
-            {
-              label: 'Total Uploaded (TB)',
-              data: uploadData,
-              borderColor: 'rgba(54, 162, 235, 1)',
-              fill: false,
-            },
-            {
-              label: 'Upload Speed (MiB/s)',
-              data: currentUploadData,
-              borderColor: 'rgba(255, 206, 86, 1)',
-              fill: false,
-            },
-            {
-              label: 'Download Speed (MiB/s)',
-              data: downloadData,
-              borderColor: 'rgba(75, 192, 192, 1)',
-              fill: false,
-            },
-            {
-              label: 'Points',
-              data: pointsData,
-              borderColor: 'rgba(153, 102, 255, 1)',
-              fill: false,
-            },
-            {
-              label: 'Seeding Count',
-              data: seedingCount,
-              borderColor: 'rgba(255, 159, 64, 1)',
-              fill: false,
-            },
+            { label: 'Rank', data: rankData, borderColor: 'rgba(255, 99, 132, 1)', fill: false },
+            { label: 'Total Uploaded (TB)', data: uploadData, borderColor: 'rgba(54, 162, 235, 1)', fill: false },
+            { label: 'Points', data: pointsData, borderColor: 'rgba(153, 102, 255, 1)', fill: false },
+            { label: 'Seeding Count', data: seedingCount, borderColor: 'rgba(255, 159, 64, 1)', fill: false },
           ],
-        }
+        };
 
         if (historyChart) {
-          historyChart.destroy()
+          historyChart.destroy();
         }
 
-        const ctx = document.getElementById('historyChart').getContext('2d')
-        historyChart = new Chart(ctx, {
-          type: 'line',
-          data: chartData,
-          options: {
-            responsive: true,
-            scales: {
-              x: {
-                title: {
-                  display: true,
-                  text: 'Date',
-                },
-              },
-              y: {
-                title: {
-                  display: true,
-                  text: 'Value',
-                },
-                beginAtZero: true,
-              },
-            },
-            plugins: {
-              tooltip: {
-                callbacks: {
-                  label: function (tooltipItem) {
-                    const label = tooltipItem.dataset.label || ''
-                    const value = tooltipItem.raw
-                    return `${label}: ${value}`
-                  },
-                },
-              },
-            },
-          },
-        })
+        const ctx = document.getElementById('historyChart').getContext('2d');
+        historyChart = new Chart(ctx, { type: 'line', data: chartData, options: { responsive: true } });
 
-        historyModal.classList.remove('hidden')
-        setTimeout(() => historyModal.classList.add('visible'), 10)
+        historyModal.classList.remove('hidden');
+        setTimeout(() => historyModal.classList.add('visible'), 10);
+      }
+
+      // Helper functions remain the same
+      function parseStorageValue(value) {
+        const numericValue = parseFloat(value);
+        if (value.includes('TiB')) return numericValue.toFixed(2);
+        if (value.includes('GiB')) return (numericValue / 1024).toFixed(2);
+        return numericValue.toFixed(2);
       }
 
       function closeModal() {
-        const historyModal = document.getElementById('historyModal')
-        historyModal.classList.remove('visible')
-        historyModal.classList.add('hidden')
+        const historyModal = document.getElementById('historyModal');
+        historyModal.classList.remove('visible');
+        historyModal.classList.add('hidden');
       }
 
-      fetchProfiles()
+      // Initial load
+      fetchProfiles();
     </script>
   </body>
 </html>
diff --git a/main.go b/main.go
index 736b94f..03fdb8f 100644
--- a/main.go
+++ b/main.go
@@ -1,9 +1,10 @@
 package main
 
 import (
+	"database/sql"
 	"encoding/json"
+	"flag"
 	"fmt"
-	"io"
 	"log"
 	"net/http"
 	"os"
@@ -13,8 +14,8 @@ import (
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
-	"github.com/fsnotify/fsnotify"
 	"github.com/joho/godotenv"
+	_ "github.com/mattn/go-sqlite3"
 )
 
 type ProfileData struct {
@@ -28,58 +29,232 @@ type ProfileData struct {
 	SeedingCount    int       `json:"seeding_count"`
 }
 
+type User struct {
+	ID          int
+	DisplayName string
+	ProfileID   string
+}
+
 var (
-	profiles     = map[string]string{}
-	jsonFile     = "./data/data.json"
-	profilesFile = "./data/profiles.json"
-	baseUrl      = "https://ncore.pro/profile.php?id="
-	nick         string
-	pass         string
-	client       *http.Client
+	db      *sql.DB
+	dbFile  = "./data/ncore_stats.db"
+	baseUrl = "https://ncore.pro/profile.php?id="
+	nick    string
+	pass    string
+	client  *http.Client
 )
 
-func init() {
+func profilesHandler(w http.ResponseWriter, r *http.Request) {
+	query := `
+	SELECT u.display_name, ph.timestamp, ph.rank, ph.upload, ph.current_upload, ph.current_download, ph.points, ph.seeding_count
+	FROM profile_history ph
+	INNER JOIN (
+		SELECT user_id, MAX(timestamp) as max_ts
+		FROM profile_history
+		GROUP BY user_id
+	) latest ON ph.user_id = latest.user_id AND ph.timestamp = latest.max_ts
+	JOIN users u ON ph.user_id = u.id;
+	`
+	rows, err := db.Query(query)
+	if err != nil {
+		http.Error(w, "Could not read latest profiles from database", http.StatusInternalServerError)
+		log.Printf("Error querying latest profiles: %v", err)
+		return
+	}
+	defer rows.Close()
+
+	var latestProfiles []ProfileData
+	for rows.Next() {
+		var p ProfileData
+		if err := rows.Scan(&p.Owner, &p.Timestamp, &p.Rank, &p.Upload, &p.CurrentUpload, &p.CurrentDownload, &p.Points, &p.SeedingCount); err != nil {
+			http.Error(w, "Could not process profile data", http.StatusInternalServerError)
+			log.Printf("Error scanning latest profile row: %v", err)
+			return
+		}
+		latestProfiles = append(latestProfiles, p)
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(latestProfiles); err != nil {
+		log.Printf("Error encoding latest profiles to JSON: %v", err)
+	}
+}
+
+func historyHandler(w http.ResponseWriter, r *http.Request) {
+	owner := r.URL.Query().Get("owner")
+	if owner == "" {
+		http.Error(w, "Missing 'owner' query parameter", http.StatusBadRequest)
+		return
+	}
+
+	query := `
+		SELECT u.display_name, ph.timestamp, ph.rank, ph.upload, ph.current_upload, ph.current_download, ph.points, ph.seeding_count
+		FROM profile_history ph
+		JOIN users u ON ph.user_id = u.id
+		WHERE u.display_name = ?
+		ORDER BY ph.timestamp ASC
+	`
+	rows, err := db.Query(query, owner)
+	if err != nil {
+		http.Error(w, "Could not read history from database", http.StatusInternalServerError)
+		log.Printf("Error querying history for %s: %v", owner, err)
+		return
+	}
+	defer rows.Close()
+
+	var userHistory []ProfileData
+	for rows.Next() {
+		var p ProfileData
+		if err := rows.Scan(&p.Owner, &p.Timestamp, &p.Rank, &p.Upload, &p.CurrentUpload, &p.CurrentDownload, &p.Points, &p.SeedingCount); err != nil {
+			http.Error(w, "Could not process history data", http.StatusInternalServerError)
+			log.Printf("Error scanning history row for %s: %v", owner, err)
+			return
+		}
+		userHistory = append(userHistory, p)
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(userHistory); err != nil {
+		log.Printf("Error encoding history for %s to JSON: %v", owner, err)
+	}
+}
+
+func main() {
+	// --- Environment and Initialization ---
 	_ = godotenv.Load(".env.local")
 	godotenv.Load()
-
 	nick = os.Getenv("NICK")
 	pass = os.Getenv("PASS")
 
-	if _, err := os.Stat(profilesFile); os.IsNotExist(err) {
-		log.Printf("File %s does not exist, creating an empty one", profilesFile)
-		err := os.WriteFile(profilesFile, []byte("{}"), 0644)
-		if err != nil {
-			log.Fatalf("Failed to create profiles file: %v", err)
-		}
+	// --- Ensure critical env vars are set ---
+	if nick == "" || pass == "" {
+		log.Fatal("FATAL: Critical environment variables NICK and/or PASS are not set. Please create a .env or .env.local file with these values.")
 	}
 
-	loadProfiles()
-
 	client = &http.Client{}
+
+	initDB()
+	defer db.Close()
+
+	// --- Command-line flags for user management ---
+	addUserFlag := flag.String("add-user", "", "Add a new user. Provide as 'DisplayName,ProfileID'")
+	flag.Parse()
+
+	if *addUserFlag != "" {
+		parts := strings.Split(*addUserFlag, ",")
+		if len(parts) != 2 {
+			log.Fatal("Invalid format for --add-user. Use 'DisplayName,ProfileID'")
+		}
+		addUser(parts[0], parts[1])
+		return // Exit after adding user
+	}
+
+	// --- Background task and Web Server ---
+	go func() {
+		fetchAndLogProfiles()
+		ticker := time.NewTicker(time.Hour * 24)
+		defer ticker.Stop()
+		for range ticker.C {
+			fetchAndLogProfiles()
+		}
+	}()
+
+	// --- Handler Registration ---
+	http.HandleFunc("/api/profiles", profilesHandler)
+	http.HandleFunc("/api/history", historyHandler)
+
+	http.HandleFunc("/", serveHTML)
+
+	log.Println("Server is starting on port 3000...")
+	log.Fatal(http.ListenAndServe(":3000", nil))
 }
 
-func loadProfiles() {
-	file, err := os.Open(profilesFile)
+func initDB() {
+	var err error
+	if err := os.MkdirAll("./data", 0755); err != nil {
+		log.Fatalf("Failed to create data directory: %v", err)
+	}
+	db, err = sql.Open("sqlite3", dbFile)
 	if err != nil {
-		log.Fatalf("Failed to open profiles file: %v", err)
+		log.Fatalf("Failed to open database: %v", err)
 	}
-	defer file.Close()
-
-	jsonBytes, err := io.ReadAll(file)
+	usersTableSQL := `CREATE TABLE IF NOT EXISTS users ("id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, "display_name" TEXT NOT NULL UNIQUE, "profile_id" TEXT NOT NULL);`
+	_, err = db.Exec(usersTableSQL)
 	if err != nil {
-		log.Fatalf("Failed to read profiles file: %v", err)
+		log.Fatalf("Failed to create users table: %v", err)
 	}
-
-	var tempProfiles map[string]string
-	err = json.Unmarshal(jsonBytes, &tempProfiles)
+	profileHistoryTableSQL := `CREATE TABLE IF NOT EXISTS profile_history ("id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, "user_id" INTEGER NOT NULL, "timestamp" DATETIME NOT NULL, "rank" INTEGER, "upload" TEXT, "current_upload" TEXT, "current_download" TEXT, "points" INTEGER, "seeding_count" INTEGER, FOREIGN KEY(user_id) REFERENCES users(id) ON DELETE CASCADE);`
+	_, err = db.Exec(profileHistoryTableSQL)
 	if err != nil {
-		log.Fatalf("Failed to unmarshal profiles JSON: %v", err)
+		log.Fatalf("Failed to create profile_history table: %v", err)
 	}
+	log.Println("Database initialized successfully.")
+}
 
-	for k, v := range tempProfiles {
-		profiles[k] = baseUrl + v
+func addUser(displayName, profileID string) {
+	stmt, err := db.Prepare("INSERT INTO users(display_name, profile_id) VALUES(?, ?)")
+	if err != nil {
+		log.Fatalf("Failed to prepare statement for adding user: %v", err)
 	}
-	log.Println("Profiles loaded successfully.")
+	defer stmt.Close()
+	_, err = stmt.Exec(displayName, profileID)
+	if err != nil {
+		log.Fatalf("Failed to add user %s: %v", displayName, err)
+	}
+	log.Printf("User '%s' with profile ID '%s' added successfully.", displayName, profileID)
+}
+
+func getUsers() ([]User, error) {
+	rows, err := db.Query("SELECT id, display_name, profile_id FROM users")
+	if err != nil {
+		return nil, fmt.Errorf("error querying users: %v", err)
+	}
+	defer rows.Close()
+	var users []User
+	for rows.Next() {
+		var u User
+		if err := rows.Scan(&u.ID, &u.DisplayName, &u.ProfileID); err != nil {
+			return nil, fmt.Errorf("error scanning user row: %v", err)
+		}
+		users = append(users, u)
+	}
+	return users, nil
+}
+
+func logToDB(profile *ProfileData, userID int) error {
+	stmt, err := db.Prepare(`INSERT INTO profile_history(user_id, timestamp, rank, upload, current_upload, current_download, points, seeding_count) VALUES(?, ?, ?, ?, ?, ?, ?, ?)`)
+	if err != nil {
+		return fmt.Errorf("error preparing insert statement: %v", err)
+	}
+	defer stmt.Close()
+	_, err = stmt.Exec(userID, profile.Timestamp, profile.Rank, profile.Upload, profile.CurrentUpload, profile.CurrentDownload, profile.Points, profile.SeedingCount)
+	if err != nil {
+		return fmt.Errorf("error executing insert for %s: %v", profile.Owner, err)
+	}
+	log.Printf("Profile for %s logged successfully to database.", profile.Owner)
+	return nil
+}
+
+func fetchAndLogProfiles() {
+	users, err := getUsers()
+	if err != nil {
+		log.Printf("Could not get users to fetch: %v", err)
+		return
+	}
+	log.Printf("Starting profile fetch for %d user(s).", len(users))
+	for _, user := range users {
+		profileURL := baseUrl + user.ProfileID
+		profile, err := fetchProfile(profileURL, user.DisplayName)
+		if err != nil {
+			log.Printf("Error fetching profile for %s: %v", user.DisplayName, err)
+			continue
+		}
+		if err := logToDB(profile, user.ID); err != nil {
+			log.Printf("Error logging profile to DB for %s: %v", user.DisplayName, err)
+		}
+		time.Sleep(2 * time.Second)
+	}
+	log.Println("Profile fetch cycle complete.")
 }
 
 func fetchProfile(url string, displayName string) (*ProfileData, error) {
@@ -87,40 +262,25 @@ func fetchProfile(url string, displayName string) (*ProfileData, error) {
 	if err != nil {
 		return nil, fmt.Errorf("error creating request for %s: %v", displayName, err)
 	}
-
 	req.Header.Set("Cookie", fmt.Sprintf("nick=%s; pass=%s", nick, pass))
-
 	resp, err := client.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("error fetching profile for %s: %v", displayName, err)
 	}
 	defer resp.Body.Close()
-
 	if resp.StatusCode != http.StatusOK {
 		return nil, fmt.Errorf("failed to fetch profile %s: received status %d", displayName, resp.StatusCode)
 	}
-
 	doc, err := goquery.NewDocumentFromReader(resp.Body)
 	if err != nil {
 		return nil, fmt.Errorf("error parsing profile document for %s: %v", displayName, err)
 	}
-
-	profile := &ProfileData{
-		Owner:     displayName,
-		Timestamp: time.Now(),
-	}
-
+	profile := &ProfileData{Owner: displayName, Timestamp: time.Now()}
 	doc.Find(".userbox_tartalom_mini .profil_jobb_elso2").Each(func(i int, s *goquery.Selection) {
-		label := s.Text()
-		value := s.Next().Text()
-
+		label, value := s.Text(), s.Next().Text()
 		switch label {
 		case "Helyezés:":
-			value = strings.TrimSuffix(value, ".")
-			rank, err := strconv.Atoi(value)
-			if err == nil {
-				profile.Rank = rank
-			}
+			profile.Rank, _ = strconv.Atoi(strings.TrimSuffix(value, "."))
 		case "Feltöltés:":
 			profile.Upload = value
 		case "Aktuális feltöltés:":
@@ -128,165 +288,21 @@ func fetchProfile(url string, displayName string) (*ProfileData, error) {
 		case "Aktuális letöltés:":
 			profile.CurrentDownload = value
 		case "Pontok száma:":
-			points, err := strconv.Atoi(value)
-			if err == nil {
-				profile.Points = points
-			}
+			profile.Points, _ = strconv.Atoi(strings.ReplaceAll(value, " ", ""))
 		}
 	})
-
 	doc.Find(".lista_mini_fej").Each(func(i int, s *goquery.Selection) {
-		text := s.Text()
-		re := regexp.MustCompile(`\((\d+)\)`)
-		matches := re.FindStringSubmatch(text)
-		if len(matches) > 1 {
+		if matches := regexp.MustCompile(`\((\d+)\)`).FindStringSubmatch(s.Text()); len(matches) > 1 {
 			fmt.Sscanf(matches[1], "%d", &profile.SeedingCount)
 		}
 	})
-
 	return profile, nil
 }
 
-func readExistingProfiles() ([]ProfileData, error) {
-	if _, err := os.Stat(jsonFile); os.IsNotExist(err) {
-		log.Printf("File %s does not exist, returning an empty profile list.", jsonFile)
-		return []ProfileData{}, nil
-	}
-
-	file, err := os.Open(jsonFile)
-	if err != nil {
-		return nil, fmt.Errorf("error opening %s: %v", jsonFile, err)
-	}
-	defer file.Close()
-
-	var existingProfiles []ProfileData
-	byteValue, err := io.ReadAll(file)
-	if err != nil {
-		return nil, fmt.Errorf("error reading %s: %v", jsonFile, err)
-	}
-
-	err = json.Unmarshal(byteValue, &existingProfiles)
-	if err != nil {
-		return nil, fmt.Errorf("error unmarshalling profile data: %v", err)
-	}
-	return existingProfiles, nil
-}
-
-func logToJSON(profile *ProfileData) error {
-	existingProfiles, err := readExistingProfiles()
-	if err != nil {
-		return err
-	}
-
-	existingProfiles = append(existingProfiles, *profile)
-
-	file, err := os.OpenFile(jsonFile, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
-	if err != nil {
-		return fmt.Errorf("error opening file for writing: %v", err)
-	}
-	defer file.Close()
-
-	enc := json.NewEncoder(file)
-	enc.SetIndent("", "  ")
-	err = enc.Encode(existingProfiles)
-	if err != nil {
-		return fmt.Errorf("error encoding JSON data: %v", err)
-	}
-	log.Printf("Profile for %s logged successfully.", profile.Owner)
-	return nil
-}
-
-func dataHandler(w http.ResponseWriter, r *http.Request) {
-	existingProfiles, err := readExistingProfiles()
-	if err != nil {
-		http.Error(w, "Could not read data", http.StatusInternalServerError)
-		log.Printf("Error reading profiles: %v", err)
+func serveHTML(w http.ResponseWriter, r *http.Request) {
+	if r.URL.Path != "/" {
+		http.NotFound(w, r)
 		return
 	}
-
-	w.Header().Set("Content-Type", "application/json")
-	err = json.NewEncoder(w).Encode(existingProfiles)
-	if err != nil {
-		log.Printf("Error encoding profiles to JSON: %v", err)
-	}
-}
-
-func serveHTML(w http.ResponseWriter, r *http.Request) {
 	http.ServeFile(w, r, "index.html")
 }
-
-func watchProfilesFile() {
-	watcher, err := fsnotify.NewWatcher()
-	if err != nil {
-		log.Fatalf("Error creating file watcher: %v", err)
-	}
-	defer watcher.Close()
-
-	err = watcher.Add(profilesFile)
-	if err != nil {
-		log.Fatalf("Error adding file to watcher: %v", err)
-	}
-
-	debounce := time.AfterFunc(0, func() {})
-
-	for {
-		select {
-		case event, ok := <-watcher.Events:
-			if !ok {
-				return
-			}
-			if event.Op&fsnotify.Write == fsnotify.Write || event.Op&fsnotify.Create == fsnotify.Create {
-				log.Println("Profiles file changed, reloading profiles...")
-				debounce.Stop()
-				debounce = time.AfterFunc(500*time.Millisecond, func() {
-					loadProfiles()
-					for displayName, url := range profiles {
-						profile, err := fetchProfile(url, displayName)
-						if err != nil {
-							log.Printf("Error fetching profile for %s after file change: %v", displayName, err)
-							continue
-						}
-
-						if err := logToJSON(profile); err != nil {
-							log.Printf("Error logging profile for %s: %v", displayName, err)
-						}
-					}
-				})
-			}
-		case err, ok := <-watcher.Errors:
-			if !ok {
-				return
-			}
-			log.Printf("File watcher error: %v", err)
-		}
-	}
-}
-
-func main() {
-	ticker := time.NewTicker(time.Hour * 24)
-	defer ticker.Stop()
-
-	go func() {
-		for {
-			for displayName, url := range profiles {
-				profile, err := fetchProfile(url, displayName)
-				if err != nil {
-					log.Printf("Error fetching profile %s: %v", displayName, err)
-					continue
-				}
-
-				if err := logToJSON(profile); err != nil {
-					log.Printf("Error logging profile for %s: %v", displayName, err)
-				}
-			}
-			<-ticker.C
-		}
-	}()
-
-	go watchProfilesFile()
-
-	http.HandleFunc("/data", dataHandler)
-	http.HandleFunc("/", serveHTML)
-	log.Println("Server is starting on port 3000...")
-	log.Fatal(http.ListenAndServe(":3000", nil))
-}
diff --git a/migrate/migrate.go b/migrate/migrate.go
new file mode 100644
index 0000000..48fe2fe
--- /dev/null
+++ b/migrate/migrate.go
@@ -0,0 +1,174 @@
+package main
+
+import (
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"time"
+
+	_ "github.com/mattn/go-sqlite3"
+)
+
+// ProfileData struct to match the structure in data.json
+type ProfileData struct {
+	Owner           string    `json:"owner"`
+	Timestamp       time.Time `json:"timestamp"`
+	Rank            int       `json:"rank"`
+	Upload          string    `json:"upload"`
+	CurrentUpload   string    `json:"current_upload"`
+	CurrentDownload string    `json:"current_download"`
+	Points          int       `json:"points"`
+	SeedingCount    int       `json:"seeding_count"`
+}
+
+var (
+	db           *sql.DB
+	dbFile       = "../data/ncore_stats.db"
+	jsonFile     = "../data/data.json"
+	profilesFile = "../data/profiles.json"
+)
+
+// initDB is the same function from the main app to set up the database.
+func initDB() {
+	var err error
+	if err := os.MkdirAll("./data", 0755); err != nil {
+		log.Fatalf("Failed to create data directory: %v", err)
+	}
+	db, err = sql.Open("sqlite3", dbFile)
+	if err != nil {
+		log.Fatalf("Failed to open database: %v", err)
+	}
+	// Create users table
+	usersTableSQL := `
+	CREATE TABLE IF NOT EXISTS users (
+		"id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+		"display_name" TEXT NOT NULL UNIQUE,
+		"profile_id" TEXT NOT NULL
+	);`
+	if _, err = db.Exec(usersTableSQL); err != nil {
+		log.Fatalf("Failed to create users table: %v", err)
+	}
+	// Create profile_history table
+	profileHistoryTableSQL := `
+	CREATE TABLE IF NOT EXISTS profile_history (
+		"id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+		"user_id" INTEGER NOT NULL,
+		"timestamp" DATETIME NOT NULL,
+		"rank" INTEGER,
+		"upload" TEXT,
+		"current_upload" TEXT,
+		"current_download" TEXT,
+		"points" INTEGER,
+		"seeding_count" INTEGER,
+		FOREIGN KEY(user_id) REFERENCES users(id) ON DELETE CASCADE
+	);`
+	if _, err = db.Exec(profileHistoryTableSQL); err != nil {
+		log.Fatalf("Failed to create profile_history table: %v", err)
+	}
+	log.Println("Database initialized successfully.")
+}
+
+func main() {
+	log.Println("Starting data migration...")
+
+	// 1. Initialize the database and tables
+	initDB()
+	defer db.Close()
+
+	// 2. Begin a transaction
+	tx, err := db.Begin()
+	if err != nil {
+		log.Fatalf("Failed to begin transaction: %v", err)
+	}
+	// Defer a rollback. If the transaction is committed, this is a no-op.
+	// If something fails, this will undo all changes.
+	defer tx.Rollback()
+
+	// --- Migrate Users ---
+	log.Println("Reading profiles.json...")
+	profFile, err := os.Open(profilesFile)
+	if err != nil {
+		log.Fatalf("Failed to open profiles file: %v", err)
+	}
+	defer profFile.Close()
+
+	var profiles map[string]string
+	if err := json.NewDecoder(profFile).Decode(&profiles); err != nil {
+		log.Fatalf("Failed to decode profiles.json: %v", err)
+	}
+
+	// This map will hold the new database ID for each user display name
+	displayNameToID := make(map[string]int64)
+
+	userStmt, err := tx.Prepare("INSERT INTO users(display_name, profile_id) VALUES(?, ?)")
+	if err != nil {
+		log.Fatalf("Failed to prepare user insert statement: %v", err)
+	}
+	defer userStmt.Close()
+
+	log.Println("Migrating users to the database...")
+	for name, id := range profiles {
+		res, err := userStmt.Exec(name, id)
+		if err != nil {
+			log.Fatalf("Failed to insert user %s: %v", name, err)
+		}
+		newID, err := res.LastInsertId()
+		if err != nil {
+			log.Fatalf("Failed to get last insert ID for user %s: %v", name, err)
+		}
+		displayNameToID[name] = newID
+		log.Printf("  > Migrated user: %s (New DB ID: %d)", name, newID)
+	}
+	log.Println("User migration complete.")
+
+	// --- Migrate History ---
+	log.Println("Reading data.json...")
+	histFile, err := os.Open(jsonFile)
+	if err != nil {
+		log.Fatalf("Failed to open data file: %v", err)
+	}
+	defer histFile.Close()
+
+	byteValue, _ := io.ReadAll(histFile)
+	var history []ProfileData
+	if err := json.Unmarshal(byteValue, &history); err != nil {
+		log.Fatalf("Failed to decode data.json: %v", err)
+	}
+
+	historyStmt, err := tx.Prepare(`
+		INSERT INTO profile_history(user_id, timestamp, rank, upload, current_upload, current_download, points, seeding_count)
+		VALUES(?, ?, ?, ?, ?, ?, ?, ?)
+	`)
+	if err != nil {
+		log.Fatalf("Failed to prepare history insert statement: %v", err)
+	}
+	defer historyStmt.Close()
+
+	log.Println("Migrating profile history to the database...")
+	for i, record := range history {
+		userID, ok := displayNameToID[record.Owner]
+		if !ok {
+			log.Printf("WARNING: Skipping history record for '%s' as they were not found in profiles.json.", record.Owner)
+			continue
+		}
+
+		_, err := historyStmt.Exec(userID, record.Timestamp, record.Rank, record.Upload, record.CurrentUpload, record.CurrentDownload, record.Points, record.SeedingCount)
+		if err != nil {
+			log.Fatalf("Failed to insert history record for %s: %v", record.Owner, err)
+		}
+		if (i+1)%100 == 0 { // Log progress every 100 records
+			log.Printf("  > Migrated %d history records...", i+1)
+		}
+	}
+	log.Println("History migration complete.")
+
+	// 3. Commit the transaction
+	if err := tx.Commit(); err != nil {
+		log.Fatalf("Failed to commit transaction: %v", err)
+	}
+
+	fmt.Println("\n✅ MIGRATION COMPLETED SUCCESSFULLY!")
+}