add repeating old requests

api/gist.js

@@ -3,11 +3,11 @@ import {
   CONSTANTS,
   renderError,
   parseBoolean,
-  storeRequest,
 } from "../src/common/utils.js";
 import { isLocaleAvailable } from "../src/translations.js";
 import { renderGistCard } from "../src/cards/gist-card.js";
 import { fetchGist } from "../src/fetchers/gist-fetcher.js";
+import { storeRequest } from "../src/common/database.js";
 
 export default async (req, res) => {
   const {

api/index.js

@@ -5,11 +5,11 @@ import {
   CONSTANTS,
   parseArray,
   parseBoolean,
-  storeRequest,
   renderError,
 } from "../src/common/utils.js";
 import { fetchStats } from "../src/fetchers/stats-fetcher.js";
 import { isLocaleAvailable } from "../src/translations.js";
+import { storeRequest } from "../src/common/database.js";
 
 export default async (req, res) => {
   const {

api/pin.js

@@ -3,13 +3,13 @@ import { blacklist } from "../src/common/blacklist.js";
 import {
   clampValue,
   CONSTANTS,
-  storeRequest,
   parseArray,
   parseBoolean,
   renderError,
 } from "../src/common/utils.js";
 import { fetchRepo } from "../src/fetchers/repo-fetcher.js";
 import { isLocaleAvailable } from "../src/translations.js";
+import { storeRequest } from "../src/common/database.js";
 
 export default async (req, res) => {
   const {

api/repeat-recent.js

@@ -0,0 +1,14 @@
+import { repeatRecentRequests } from "../src/common/database.js";
+
+export default async (req, res) => {
+  if (req.method !== "POST") {
+    res.status(405).json({ error: "Method Not Allowed" });
+    return;
+  }
+  try {
+    await repeatRecentRequests();
+    res.status(200).json({ message: "Recent requests repeated successfully." });
+  } catch (error) {
+    res.status(500).json({ error: error.message || "Internal Server Error" });
+  }
+};

api/top-langs.js

@@ -5,10 +5,10 @@ import {
   parseArray,
   parseBoolean,
   renderError,
-  storeRequest,
 } from "../src/common/utils.js";
 import { fetchTopLanguages } from "../src/fetchers/top-languages-fetcher.js";
 import { isLocaleAvailable } from "../src/translations.js";
+import { storeRequest } from "../src/common/database.js";
 
 export default async (req, res) => {
   const {

api/wakatime.js

@@ -5,10 +5,10 @@ import {
   parseArray,
   parseBoolean,
   renderError,
-  storeRequest,
 } from "../src/common/utils.js";
 import { fetchWakatimeStats } from "../src/fetchers/wakatime-fetcher.js";
 import { isLocaleAvailable } from "../src/translations.js";
+import { storeRequest } from "../src/common/database.js";
 
 export default async (req, res) => {
   const {

src/common/database.js

@@ -0,0 +1,134 @@
+import axios from "axios";
+import pkg from "pg";
+const { Pool } = pkg;
+
+const pool = process.env.POSTGRES_URL
+  ? new Pool({
+      connectionString: process.env.POSTGRES_URL,
+    })
+  : null;
+
+/**
+ * Stores or updates a request in the database.
+ */
+export async function storeRequest(req) {
+  if (!pool) {
+    return;
+  }
+
+  const insertQuery = `
+      INSERT INTO requests (request, requested_at)
+      VALUES ($1, NOW())
+      ON CONFLICT (request)
+      DO UPDATE SET requested_at = EXCLUDED.requested_at
+    `;
+
+  try {
+    await pool.query(insertQuery, [req.url]);
+  } catch (err) {
+    // Check for undefined_table error (SQLSTATE 42P01)
+    if (err.code === "42P01") {
+      const createTableQuery = `
+          CREATE TABLE IF NOT EXISTS requests (
+            request TEXT PRIMARY KEY,
+            requested_at TIMESTAMP NOT NULL DEFAULT now()
+          )
+        `;
+      await pool.query(createTableQuery);
+      // Retry the insert after creating the table
+      await pool.query(insertQuery, [req.url]);
+    } else {
+      throw err; // Re-throw if it's some other error
+    }
+  }
+}
+
+/**
+ * Deletes all requests older than 7 days from the database.
+ */
+async function deleteOldRequests() {
+  if (!pool) {
+    return;
+  }
+
+  const deleteQuery = `
+      DELETE FROM requests
+      WHERE requested_at < NOW() - INTERVAL '7 days'
+    `;
+  const result = await pool.query(deleteQuery);
+  console.log(`Deleted ${result.rowCount} old requests.`);
+}
+
+/**
+ * Fetches all requests from the last 7 days.
+ * @returns {Promise<string[]>} An array of requests made in the last 7 days.
+ */
+async function getRecentRequests() {
+  if (!pool) {
+    return [];
+  }
+
+  const query = `
+      SELECT request
+      FROM requests
+      WHERE requested_at >= NOW() - INTERVAL '7 days'
+        AND requested_at < NOW() - INTERVAL '1 hour';
+      `;
+  const { rows } = await pool.query(query);
+  return rows.map((row) => row.request);
+}
+
+/**
+ * Processes URLs with a thread pool of given size using axios.get.
+ * @param {string[]} urls An array of URLs to process.
+ * @param {number} poolSize The number of concurrent requests to process.
+ * @returns {Promise<void>} A promise that resolves when all requests are processed.
+ */
+async function makeRequests(urls, poolSize) {
+  let current = 0;
+
+  /**
+   * Worker function to process `urls`.
+   */
+  async function worker() {
+    while (true) {
+      let idx = current++;
+      if (idx >= urls.length) {
+        break;
+      }
+      const url = urls[idx];
+      try {
+        if (idx % 10 === 0) {
+          console.log(`Processing request ${idx + 1} out of ${urls.length}`);
+        }
+        await axios.get(url, { timeout: 10000 });
+      } catch (err) {
+        console.error(`Error fetching ${url}:`, err.message);
+      }
+    }
+  }
+
+  const workers = [];
+  for (let i = 0; i < poolSize; i++) {
+    workers.push(worker());
+  }
+  await Promise.all(workers);
+}
+
+/**
+ * Repeats requests made in the last 7 days, excluding those made in the last hour.
+ */
+export async function repeatRecentRequests() {
+  if (!pool) {
+    console.error("Postgres pool is not initialized.");
+    return;
+  }
+
+  await deleteOldRequests();
+  const urls = await getRecentRequests();
+  if (urls.length === 0) {
+    console.log("No recent requests found.");
+  } else {
+    await makeRequests(urls, 5);
+  }
+}

src/common/utils.js

@@ -3,14 +3,7 @@ import axios from "axios";
 import toEmoji from "emoji-name-map";
 import wrap from "word-wrap";
 import { themes } from "../../themes/index.js";
-import pkg from "pg";
-const { Pool } = pkg;
 
-const pool = process.env.POSTGRES_URL
-  ? new Pool({
-      connectionString: process.env.POSTGRES_URL,
-    })
-  : null;
 const TRY_AGAIN_LATER = "Please try again later";
 
 const SECONDARY_ERROR_MESSAGES = {
@@ -257,41 +250,6 @@ const request = (data, headers) => {
   });
 };
 
-/**
- * Stores or updates the request in the database.
- */
-export async function storeRequest(req) {
-  if (!pool) {
-    return;
-  }
-
-  const insertQuery = `
-    INSERT INTO requests (request, requested_at)
-    VALUES ($1, NOW())
-    ON CONFLICT (request)
-    DO UPDATE SET requested_at = EXCLUDED.requested_at
-  `;
-
-  try {
-    await pool.query(insertQuery, [req.url]);
-  } catch (err) {
-    // Check for undefined_table error (SQLSTATE 42P01)
-    if (err.code === "42P01") {
-      const createTableQuery = `
-        CREATE TABLE IF NOT EXISTS requests (
-          request TEXT PRIMARY KEY,
-          requested_at TIMESTAMP NOT NULL DEFAULT now()
-        )
-      `;
-      await pool.query(createTableQuery);
-      // Retry the insert after creating the table
-      await pool.query(insertQuery, [req.url]);
-    } else {
-      throw err; // Re-throw if it's some other error
-    }
-  }
-}
-
 /**
  * Object containing card colors.
  * @typedef {{