From 7cc0217f354b90c76736e3adc38e11fdf12897d0 Mon Sep 17 00:00:00 2001 From: martin-mfg <2026226+martin-mfg@users.noreply.github.com> Date: Sun, 22 Jun 2025 07:14:56 +0000 Subject: [PATCH] add repeating old requests --- api/gist.js | 2 +- api/index.js | 2 +- api/pin.js | 2 +- api/repeat-recent.js | 14 +++++ api/top-langs.js | 2 +- api/wakatime.js | 2 +- src/common/database.js | 134 +++++++++++++++++++++++++++++++++++++++++ src/common/utils.js | 42 ------------- 8 files changed, 153 insertions(+), 47 deletions(-) create mode 100644 api/repeat-recent.js create mode 100644 src/common/database.js diff --git a/api/gist.js b/api/gist.js index 482d1cc3..e72cd304 100644 --- a/api/gist.js +++ b/api/gist.js @@ -3,11 +3,11 @@ import { CONSTANTS, renderError, parseBoolean, - storeRequest, } from "../src/common/utils.js"; import { isLocaleAvailable } from "../src/translations.js"; import { renderGistCard } from "../src/cards/gist-card.js"; import { fetchGist } from "../src/fetchers/gist-fetcher.js"; +import { storeRequest } from "../src/common/database.js"; export default async (req, res) => { const { diff --git a/api/index.js b/api/index.js index 2756bd5a..ad96d594 100644 --- a/api/index.js +++ b/api/index.js @@ -5,11 +5,11 @@ import { CONSTANTS, parseArray, parseBoolean, - storeRequest, renderError, } from "../src/common/utils.js"; import { fetchStats } from "../src/fetchers/stats-fetcher.js"; import { isLocaleAvailable } from "../src/translations.js"; +import { storeRequest } from "../src/common/database.js"; export default async (req, res) => { const { diff --git a/api/pin.js b/api/pin.js index a43db7b0..c143b80f 100644 --- a/api/pin.js +++ b/api/pin.js @@ -3,13 +3,13 @@ import { blacklist } from "../src/common/blacklist.js"; import { clampValue, CONSTANTS, - storeRequest, parseArray, parseBoolean, renderError, } from "../src/common/utils.js"; import { fetchRepo } from "../src/fetchers/repo-fetcher.js"; import { isLocaleAvailable } from "../src/translations.js"; +import { storeRequest } from "../src/common/database.js"; export default async (req, res) => { const { diff --git a/api/repeat-recent.js b/api/repeat-recent.js new file mode 100644 index 00000000..148b46f1 --- /dev/null +++ b/api/repeat-recent.js @@ -0,0 +1,14 @@ +import { repeatRecentRequests } from "../src/common/database.js"; + +export default async (req, res) => { + if (req.method !== "POST") { + res.status(405).json({ error: "Method Not Allowed" }); + return; + } + try { + await repeatRecentRequests(); + res.status(200).json({ message: "Recent requests repeated successfully." }); + } catch (error) { + res.status(500).json({ error: error.message || "Internal Server Error" }); + } +}; diff --git a/api/top-langs.js b/api/top-langs.js index 05360257..fc8aef96 100644 --- a/api/top-langs.js +++ b/api/top-langs.js @@ -5,10 +5,10 @@ import { parseArray, parseBoolean, renderError, - storeRequest, } from "../src/common/utils.js"; import { fetchTopLanguages } from "../src/fetchers/top-languages-fetcher.js"; import { isLocaleAvailable } from "../src/translations.js"; +import { storeRequest } from "../src/common/database.js"; export default async (req, res) => { const { diff --git a/api/wakatime.js b/api/wakatime.js index fcf70124..b63cfe51 100644 --- a/api/wakatime.js +++ b/api/wakatime.js @@ -5,10 +5,10 @@ import { parseArray, parseBoolean, renderError, - storeRequest, } from "../src/common/utils.js"; import { fetchWakatimeStats } from "../src/fetchers/wakatime-fetcher.js"; import { isLocaleAvailable } from "../src/translations.js"; +import { storeRequest } from "../src/common/database.js"; export default async (req, res) => { const { diff --git a/src/common/database.js b/src/common/database.js new file mode 100644 index 00000000..b551e063 --- /dev/null +++ b/src/common/database.js @@ -0,0 +1,134 @@ +import axios from "axios"; +import pkg from "pg"; +const { Pool } = pkg; + +const pool = process.env.POSTGRES_URL + ? new Pool({ + connectionString: process.env.POSTGRES_URL, + }) + : null; + +/** + * Stores or updates a request in the database. + */ +export async function storeRequest(req) { + if (!pool) { + return; + } + + const insertQuery = ` + INSERT INTO requests (request, requested_at) + VALUES ($1, NOW()) + ON CONFLICT (request) + DO UPDATE SET requested_at = EXCLUDED.requested_at + `; + + try { + await pool.query(insertQuery, [req.url]); + } catch (err) { + // Check for undefined_table error (SQLSTATE 42P01) + if (err.code === "42P01") { + const createTableQuery = ` + CREATE TABLE IF NOT EXISTS requests ( + request TEXT PRIMARY KEY, + requested_at TIMESTAMP NOT NULL DEFAULT now() + ) + `; + await pool.query(createTableQuery); + // Retry the insert after creating the table + await pool.query(insertQuery, [req.url]); + } else { + throw err; // Re-throw if it's some other error + } + } +} + +/** + * Deletes all requests older than 7 days from the database. + */ +async function deleteOldRequests() { + if (!pool) { + return; + } + + const deleteQuery = ` + DELETE FROM requests + WHERE requested_at < NOW() - INTERVAL '7 days' + `; + const result = await pool.query(deleteQuery); + console.log(`Deleted ${result.rowCount} old requests.`); +} + +/** + * Fetches all requests from the last 7 days. + * @returns {Promise} An array of requests made in the last 7 days. + */ +async function getRecentRequests() { + if (!pool) { + return []; + } + + const query = ` + SELECT request + FROM requests + WHERE requested_at >= NOW() - INTERVAL '7 days' + AND requested_at < NOW() - INTERVAL '1 hour'; + `; + const { rows } = await pool.query(query); + return rows.map((row) => row.request); +} + +/** + * Processes URLs with a thread pool of given size using axios.get. + * @param {string[]} urls An array of URLs to process. + * @param {number} poolSize The number of concurrent requests to process. + * @returns {Promise} A promise that resolves when all requests are processed. + */ +async function makeRequests(urls, poolSize) { + let current = 0; + + /** + * Worker function to process `urls`. + */ + async function worker() { + while (true) { + let idx = current++; + if (idx >= urls.length) { + break; + } + const url = urls[idx]; + try { + if (idx % 10 === 0) { + console.log(`Processing request ${idx + 1} out of ${urls.length}`); + } + await axios.get(url, { timeout: 10000 }); + } catch (err) { + console.error(`Error fetching ${url}:`, err.message); + } + } + } + + const workers = []; + for (let i = 0; i < poolSize; i++) { + workers.push(worker()); + } + await Promise.all(workers); +} + +/** + * Repeats requests made in the last 7 days, excluding those made in the last hour. + */ +export async function repeatRecentRequests() { + if (!pool) { + console.error("Postgres pool is not initialized."); + return; + } + + await deleteOldRequests(); + const urls = await getRecentRequests(); + if (urls.length === 0) { + console.log("No recent requests found."); + } else { + await makeRequests(urls, 5); + } +} diff --git a/src/common/utils.js b/src/common/utils.js index ac514913..81ebbfcc 100644 --- a/src/common/utils.js +++ b/src/common/utils.js @@ -3,14 +3,7 @@ import axios from "axios"; import toEmoji from "emoji-name-map"; import wrap from "word-wrap"; import { themes } from "../../themes/index.js"; -import pkg from "pg"; -const { Pool } = pkg; -const pool = process.env.POSTGRES_URL - ? new Pool({ - connectionString: process.env.POSTGRES_URL, - }) - : null; const TRY_AGAIN_LATER = "Please try again later"; const SECONDARY_ERROR_MESSAGES = { @@ -257,41 +250,6 @@ const request = (data, headers) => { }); }; -/** - * Stores or updates the request in the database. - */ -export async function storeRequest(req) { - if (!pool) { - return; - } - - const insertQuery = ` - INSERT INTO requests (request, requested_at) - VALUES ($1, NOW()) - ON CONFLICT (request) - DO UPDATE SET requested_at = EXCLUDED.requested_at - `; - - try { - await pool.query(insertQuery, [req.url]); - } catch (err) { - // Check for undefined_table error (SQLSTATE 42P01) - if (err.code === "42P01") { - const createTableQuery = ` - CREATE TABLE IF NOT EXISTS requests ( - request TEXT PRIMARY KEY, - requested_at TIMESTAMP NOT NULL DEFAULT now() - ) - `; - await pool.query(createTableQuery); - // Retry the insert after creating the table - await pool.query(insertQuery, [req.url]); - } else { - throw err; // Re-throw if it's some other error - } - } -} - /** * Object containing card colors. * @typedef {{