From f9791457683156a8f33f1d2ccb077a711496b31f Mon Sep 17 00:00:00 2001 From: martin-mfg <2026226+martin-mfg@users.noreply.github.com> Date: Sat, 21 Jun 2025 13:17:53 +0000 Subject: [PATCH] add DB logging --- api/gist.js | 2 + api/index.js | 2 + api/pin.js | 2 + api/top-langs.js | 2 + api/wakatime.js | 2 + express.js | 2 + jest.config.js | 1 + package-lock.json | 242 ++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + src/common/utils.js | 39 +++++++ tests/setup.jest.js | 5 + 11 files changed, 300 insertions(+) create mode 100644 tests/setup.jest.js diff --git a/api/gist.js b/api/gist.js index 5b03743c..482d1cc3 100644 --- a/api/gist.js +++ b/api/gist.js @@ -3,6 +3,7 @@ import { CONSTANTS, renderError, parseBoolean, + storeRequest, } from "../src/common/utils.js"; import { isLocaleAvailable } from "../src/translations.js"; import { renderGistCard } from "../src/cards/gist-card.js"; @@ -39,6 +40,7 @@ export default async (req, res) => { } try { + await storeRequest(req); const gistData = await fetchGist(id); let cacheSeconds = clampValue( diff --git a/api/index.js b/api/index.js index 6b14bf58..2756bd5a 100644 --- a/api/index.js +++ b/api/index.js @@ -5,6 +5,7 @@ import { CONSTANTS, parseArray, parseBoolean, + storeRequest, renderError, } from "../src/common/utils.js"; import { fetchStats } from "../src/fetchers/stats-fetcher.js"; @@ -89,6 +90,7 @@ export default async (req, res) => { } try { + await storeRequest(req); const showStats = parseArray(show); const organizations = parseArray(owners); let repositories = parseArray(repos); diff --git a/api/pin.js b/api/pin.js index 2570eeb6..a43db7b0 100644 --- a/api/pin.js +++ b/api/pin.js @@ -3,6 +3,7 @@ import { blacklist } from "../src/common/blacklist.js"; import { clampValue, CONSTANTS, + storeRequest, parseArray, parseBoolean, renderError, @@ -81,6 +82,7 @@ export default async (req, res) => { } try { + await storeRequest(req); const showStats = parseArray(show); const repoData = await fetchRepo( username, diff --git a/api/top-langs.js b/api/top-langs.js index c5bed634..05360257 100644 --- a/api/top-langs.js +++ b/api/top-langs.js @@ -5,6 +5,7 @@ import { parseArray, parseBoolean, renderError, + storeRequest, } from "../src/common/utils.js"; import { fetchTopLanguages } from "../src/fetchers/top-languages-fetcher.js"; import { isLocaleAvailable } from "../src/translations.js"; @@ -62,6 +63,7 @@ export default async (req, res) => { } try { + await storeRequest(req); const topLangs = await fetchTopLanguages( username, parseArray(exclude_repo), diff --git a/api/wakatime.js b/api/wakatime.js index 73ef9986..fcf70124 100644 --- a/api/wakatime.js +++ b/api/wakatime.js @@ -5,6 +5,7 @@ import { parseArray, parseBoolean, renderError, + storeRequest, } from "../src/common/utils.js"; import { fetchWakatimeStats } from "../src/fetchers/wakatime-fetcher.js"; import { isLocaleAvailable } from "../src/translations.js"; @@ -49,6 +50,7 @@ export default async (req, res) => { } try { + await storeRequest(req); const stats = await fetchWakatimeStats({ username, api_domain }); let cacheSeconds = clampValue( diff --git a/express.js b/express.js index ddc69ba7..8fd9b3c4 100644 --- a/express.js +++ b/express.js @@ -5,7 +5,9 @@ import langCard from "./api/top-langs.js"; import wakatimeCard from "./api/wakatime.js"; import gistCard from "./api/gist.js"; import express from "express"; +import ensureDbSchema from "./src/common/utils.js"; +ensureDbSchema(); const app = express(); app.listen(process.env.port || 9000); diff --git a/jest.config.js b/jest.config.js index b4578cf5..8b2eb258 100644 --- a/jest.config.js +++ b/jest.config.js @@ -3,6 +3,7 @@ export default { transform: {}, testEnvironment: "jsdom", coverageProvider: "v8", + setupFiles: ["/tests/setup.jest.js"], testPathIgnorePatterns: ["/node_modules/", "/tests/e2e/"], modulePathIgnorePatterns: ["/node_modules/", "/tests/e2e/"], coveragePathIgnorePatterns: [ diff --git a/package-lock.json b/package-lock.json index dc391590..595a1a41 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "dotenv": "^16.5.0", "emoji-name-map": "^1.2.8", "github-username-regex": "^1.0.0", + "pg": "^8.16.2", "upgrade": "^1.1.0", "word-wrap": "^1.2.5" }, @@ -6526,6 +6527,104 @@ "node": ">=16" } }, + "node_modules/pg": { + "version": "8.16.2", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.16.2.tgz", + "integrity": "sha512-OtLWF0mKLmpxelOt9BqVq83QV6bTfsS0XLegIeAKqKjurRnRKie1Dc1iL89MugmSLhftxw6NNCyZhm1yQFLMEQ==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.9.1", + "pg-pool": "^3.10.1", + "pg-protocol": "^1.10.2", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.2.6" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.2.6.tgz", + "integrity": "sha512-uxmJAnmIgmYgnSFzgOf2cqGQBzwnRYcrEgXuFjJNEkpedEIPBSEzxY7ph4uA9k1mI+l/GR0HjPNS6FKNZe8SBQ==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.9.1.tgz", + "integrity": "sha512-nkc6NpDcvPVpZXxrreI/FOtX3XemeLl8E0qFr6F2Lrm/I8WOnaWNhIPK2Z7OHpw7gh5XJThi6j6ppgNoaT1w4w==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.10.1.tgz", + "integrity": "sha512-Tu8jMlcX+9d8+QVzKIvM/uJtp07PKr82IUOYEphaWcoBhIYkoHpLXN3qO59nAI11ripznDsEzEv8nUxBVWajGg==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.10.2.tgz", + "integrity": "sha512-Ci7jy8PbaWxfsck2dwZdERcDG2A0MG8JoQILs+uZNjABFuBuItAZCWUNz8sXRDMoui24rJw7WlXqgpMdBSN/vQ==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/pgpass/node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/picocolors": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", @@ -6593,6 +6692,45 @@ "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", "dev": true }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", @@ -7858,6 +7996,15 @@ "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", "dev": true }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", @@ -12705,6 +12852,73 @@ "integrity": "sha512-TdrF7fW9Rphjq4RjrW0Kp2AW0Ahwu9sRGTkS6bvDi0SCwZlEZYmcfDbEsTz8RVk0EHIS/Vd1bv3JhG+1xZuAyQ==", "dev": true }, + "pg": { + "version": "8.16.2", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.16.2.tgz", + "integrity": "sha512-OtLWF0mKLmpxelOt9BqVq83QV6bTfsS0XLegIeAKqKjurRnRKie1Dc1iL89MugmSLhftxw6NNCyZhm1yQFLMEQ==", + "requires": { + "pg-cloudflare": "^1.2.6", + "pg-connection-string": "^2.9.1", + "pg-pool": "^3.10.1", + "pg-protocol": "^1.10.2", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + } + }, + "pg-cloudflare": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.2.6.tgz", + "integrity": "sha512-uxmJAnmIgmYgnSFzgOf2cqGQBzwnRYcrEgXuFjJNEkpedEIPBSEzxY7ph4uA9k1mI+l/GR0HjPNS6FKNZe8SBQ==", + "optional": true + }, + "pg-connection-string": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.9.1.tgz", + "integrity": "sha512-nkc6NpDcvPVpZXxrreI/FOtX3XemeLl8E0qFr6F2Lrm/I8WOnaWNhIPK2Z7OHpw7gh5XJThi6j6ppgNoaT1w4w==" + }, + "pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==" + }, + "pg-pool": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.10.1.tgz", + "integrity": "sha512-Tu8jMlcX+9d8+QVzKIvM/uJtp07PKr82IUOYEphaWcoBhIYkoHpLXN3qO59nAI11ripznDsEzEv8nUxBVWajGg==", + "requires": {} + }, + "pg-protocol": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.10.2.tgz", + "integrity": "sha512-Ci7jy8PbaWxfsck2dwZdERcDG2A0MG8JoQILs+uZNjABFuBuItAZCWUNz8sXRDMoui24rJw7WlXqgpMdBSN/vQ==" + }, + "pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "requires": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + } + }, + "pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "requires": { + "split2": "^4.1.0" + }, + "dependencies": { + "split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==" + } + } + }, "picocolors": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", @@ -12750,6 +12964,29 @@ "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", "dev": true }, + "postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==" + }, + "postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==" + }, + "postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==" + }, + "postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "requires": { + "xtend": "^4.0.0" + } + }, "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", @@ -13651,6 +13888,11 @@ "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", "dev": true }, + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + }, "y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index b70eac1e..470660ed 100644 --- a/package.json +++ b/package.json @@ -65,6 +65,7 @@ "dotenv": "^16.5.0", "emoji-name-map": "^1.2.8", "github-username-regex": "^1.0.0", + "pg": "^8.16.2", "upgrade": "^1.1.0", "word-wrap": "^1.2.5" }, diff --git a/src/common/utils.js b/src/common/utils.js index 81ebbfcc..2cf89cca 100644 --- a/src/common/utils.js +++ b/src/common/utils.js @@ -3,7 +3,14 @@ import axios from "axios"; import toEmoji from "emoji-name-map"; import wrap from "word-wrap"; import { themes } from "../../themes/index.js"; +import pkg from "pg"; +const { Pool } = pkg; +const pool = process.env.POSTGRES_URL + ? new Pool({ + connectionString: process.env.POSTGRES_URL, + }) + : null; const TRY_AGAIN_LATER = "Please try again later"; const SECONDARY_ERROR_MESSAGES = { @@ -250,6 +257,38 @@ const request = (data, headers) => { }); }; +/** + * Creates the database table "requests" if it does not exist. + */ +export async function ensureDbSchema() { + if (!pool) { + return; + } + const createTableSQL = ` + CREATE TABLE IF NOT EXISTS requests ( + request TEXT PRIMARY KEY, + requested_at TIMESTAMP NOT NULL DEFAULT now() + ); + `; + await pool.query(createTableSQL); +} + +/** + * Stores or updates the request in the database. + */ +export async function storeRequest(req) { + if (!pool) { + return; + } + const query = ` + INSERT INTO requests (request, requested_at) + VALUES ($1, NOW()) + ON CONFLICT (request) + DO UPDATE SET requested_at = EXCLUDED.requested_at + `; + await pool.query(query, [req.url]); +} + /** * Object containing card colors. * @typedef {{ diff --git a/tests/setup.jest.js b/tests/setup.jest.js new file mode 100644 index 00000000..08947667 --- /dev/null +++ b/tests/setup.jest.js @@ -0,0 +1,5 @@ +//https://stackoverflow.com/a/68468204/1643179 + +import { TextEncoder, TextDecoder } from "util"; + +Object.assign(global, { TextDecoder, TextEncoder });