initiate monorepo, add original github-trends

backend/src/repeatRequests.js

@@ -0,0 +1,65 @@
+import axios from "axios";
+import {
+  pool,
+  deleteOldRequests,
+  getRecentRequests,
+} from "./common/database.js";
+
+/**
+ * Processes URLs with a thread pool of given size using axios.get.
+ *
+ * @param {string[]} urls An array of URLs to process.
+ * @param {number} poolSize The number of concurrent requests to process.
+ * @returns {Promise<void>} A promise that resolves when all requests are processed.
+ */
+async function makeRequests(urls, poolSize) {
+  let current = 0;
+
+  /**
+   * Worker function to process `urls`.
+   */
+  async function worker() {
+    while (true) {
+      let idx = current++;
+      if (idx >= urls.length) {
+        break;
+      }
+      const url = "https://" + process.env.VERCEL_BRANCH_URL + urls[idx];
+      try {
+        if (idx % 10 === 0) {
+          console.log(`Processing request ${idx + 1} out of ${urls.length}`);
+        }
+        await axios.get(url, {
+          timeout: 10000,
+          headers: { "x-bypass-store": "true" },
+        });
+      } catch (err) {
+        console.error(`Error fetching ${url}:`, err.message);
+      }
+    }
+  }
+
+  const workers = [];
+  for (let i = 0; i < poolSize; i++) {
+    workers.push(worker());
+  }
+  await Promise.all(workers);
+}
+
+/**
+ * Repeats requests made in the last 8 days, excluding those made in the last 11 hours.
+ */
+export async function repeatRecentRequests() {
+  if (!pool) {
+    console.error("Postgres pool is not initialized.");
+    return;
+  }
+
+  await deleteOldRequests();
+  const urls = await getRecentRequests();
+  if (urls.length === 0) {
+    console.log("No recent requests found.");
+  } else {
+    await makeRequests(urls, 5);
+  }
+}