fix(nginx): return 404 for non-existent articles/ and sections/ paths

Previously nginx try_files fell back to /index.html for any unmatched path,
which meant orphaned article variants (old slug directories) returned HTTP 200
with the homepage content. This caused ~1,150 orphaned pages to be crawlable.

Add explicit location blocks for /articles/ and /sections/ that use =404
instead of falling back to /index.html.

.gitea/workflows/build-and-deploy.yaml

@@ -82,6 +82,7 @@ jobs:
       - name: Apply Kubernetes manifests
         run: |
           kubectl apply -f k8s/namespace.yaml
+          kubectl apply -f k8s/middleware.yaml
           kubectl apply -f k8s/deployment.yaml
           kubectl apply -f k8s/service.yaml
           kubectl apply -f k8s/ingress.yaml

Dockerfile

@@ -32,7 +32,13 @@ RUN echo 'server { \
     location / { \
         try_files $uri $uri/ /index.html; \
     } \
-    location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf|eot|webp|avif)$ { \
+    location /articles/ { \
+        try_files $uri $uri/ =404; \
+    } \
+    location /sections/ { \
+        try_files $uri $uri/ =404; \
+    } \
+    location ~* \\.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf|eot|webp|avif)$ { \
         expires 1y; \
         add_header Cache-Control "public, immutable"; \
     } \

README.md

@@ -1,3 +1,61 @@
 # signalledger.nl
 
-Signal Ledger news site
+Signal Ledger — an independent news publication, a subsidiary of Jopdorp.
+
+## Architecture Decision Record (ADR)
+
+### ADR-002: Ingress Controller Migration (nginx → Traefik)
+
+**Status:** Accepted
+
+**Context:**
+The cluster uses Traefik as its ingress controller. The initial K8s manifests were written with `ingressClassName: nginx` and nginx-specific annotations. This caused a mismatch: the Ingress resource was never picked up by any controller, leaving the site unreachable via the configured domains.
+
+**Decision:**
+Migrate all ingress configuration to Traefik-native resources.
+
+1. **Ingress class:** Changed from `nginx` to `traefik`.
+2. **Annotations:** Replaced nginx-specific `configuration-snippet` with Traefik `router.entrypoints`, `router.tls`, and `router.middlewares` annotations.
+3. **Security headers:** Extracted from inline nginx snippets into a dedicated `Middleware` CRD (`k8s/middleware.yaml`). This keeps header policy declarative and reusable.
+
+**Migration strategy:** In-place update
+- The namespace `openclaw-private` already exists.
+- The deployment, service, and TLS secret are unchanged.
+- We apply the new Ingress and Middleware manifests; Traefik picks them up immediately.
+- Rolling back is a single `kubectl apply` of the previous manifest version.
+
+**Consequences:**
+- Positive: Aligns with cluster infrastructure. No extra ingress controller needed.
+- Positive: Middleware CRD is cleaner and version-controllable than inline snippets.
+- Risk: Traefik middleware syntax errors will cause 404/500 until fixed. Mitigated by validating manifests in CI before deploy.
+
+## Deployment
+
+### Prerequisites
+- Kubernetes cluster with Traefik and cert-manager installed.
+- `registry.claw.jopdorp.nl` push access.
+- `KUBECONFIG_BASE64` and `REGISTRY_TOKEN` secrets configured in Gitea.
+
+### CI/CD Pipeline
+Gitea Actions workflow (`.gitea/workflows/build-and-deploy.yaml`):
+1. Build and test on every PR/push.
+2. Build and push Docker image on merge to `main`.
+3. Apply K8s manifests and wait for rollout.
+
+### Manual Deploy
+```bash
+kubectl apply -f k8s/namespace.yaml
+kubectl apply -f k8s/middleware.yaml
+kubectl apply -f k8s/deployment.yaml
+kubectl apply -f k8s/service.yaml
+kubectl apply -f k8s/ingress.yaml
+kubectl rollout status deployment/news-site -n openclaw-private --timeout=120s
+```
+
+### Domains
+- `signalledger.nl`
+- `www.signalledger.nl`
+
+### Contact
+- Email: signalledger@jopdorp.nl
+- Owner: Signal Ledger is a subsidiary of Jopdorp.

k8s/deployment.yaml

@@ -4,7 +4,7 @@ metadata:
   name: news-site
   namespace: openclaw-private
   labels:
-    app.kubernetes.io/name: signalledger
+    app: news-site
     app.kubernetes.io/component: frontend
     app.kubernetes.io/part-of: signalledger
     app.kubernetes.io/managed-by: gitea-actions
@@ -17,11 +17,11 @@ spec:
       maxUnavailable: 0
   selector:
     matchLabels:
-      app.kubernetes.io/name: signalledger
+      app: news-site
   template:
     metadata:
       labels:
-        app.kubernetes.io/name: signalledger
+        app: news-site
         app.kubernetes.io/component: frontend
         app.kubernetes.io/part-of: signalledger
     spec:

k8s/ingress.yaml

@@ -4,21 +4,33 @@ metadata:
   name: news-site
   namespace: openclaw-private
   labels:
-    app.kubernetes.io/name: signalledger
+    app: news-site
     app.kubernetes.io/component: frontend
     app.kubernetes.io/part-of: signalledger
   annotations:
     cert-manager.io/cluster-issuer: "letsencrypt-prod"
-    traefik.ingress.kubernetes.io/router.entrypoints: websecure
+    traefik.ingress.kubernetes.io/router.entrypoints: "websecure"
     traefik.ingress.kubernetes.io/router.tls: "true"
+    traefik.ingress.kubernetes.io/router.middlewares: "openclaw-private-security-headers@kubernetescrd"
 spec:
   ingressClassName: traefik
   tls:
     - hosts:
+        - news.claw.jopdorp.nl
         - signalledger.nl
         - www.signalledger.nl
-      secretName: signalledger-tls
+      secretName: news-site-tls
   rules:
+    - host: news.claw.jopdorp.nl
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: news-site
+                port:
+                  number: 80
     - host: signalledger.nl
       http:
         paths:
@@ -39,13 +51,3 @@ spec:
                 name: news-site
                 port:
                   number: 80
-    - host: news.claw.jopdorp.nl
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: news-site
-                port:
-                  number: 80

k8s/middleware.yaml

@@ -0,0 +1,22 @@
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  name: security-headers
+  namespace: openclaw-private
+  labels:
+    app: news-site
+    app.kubernetes.io/component: middleware
+    app.kubernetes.io/part-of: signalledger
+spec:
+  headers:
+    customRequestHeaders:
+      X-Forwarded-Proto: "https"
+    customResponseHeaders:
+      Strict-Transport-Security: "max-age=31536000; includeSubDomains; preload"
+      X-Frame-Options: "DENY"
+      X-Content-Type-Options: "nosniff"
+      Referrer-Policy: "strict-origin-when-cross-origin"
+      Content-Security-Policy: "default-src 'self'; script-src 'self' 'unsafe-inline' https://pagead2.googlesyndication.com https://partner.googleadservices.com https://tpc.googlesyndication.com; img-src 'self' data: https:; style-src 'self' 'unsafe-inline'; font-src 'self'; connect-src 'self'; frame-src https://googleads.g.doubleclick.net; object-src 'none'; base-uri 'self'; form-action 'self';"
+      X-XSS-Protection: "1; mode=block"
+      Permissions-Policy: "accelerometer=(), camera=(), geolocation=(), gyroscope=(), magnetometer=(), microphone=(), payment=(), usb=()"
+    sslRedirect: true

k8s/service.yaml

@@ -4,12 +4,12 @@ metadata:
   name: news-site
   namespace: openclaw-private
   labels:
-    app.kubernetes.io/name: signalledger
+    app: news-site
     app.kubernetes.io/component: frontend
     app.kubernetes.io/part-of: signalledger
 spec:
   selector:
-    app.kubernetes.io/name: signalledger
+    app: news-site
   ports:
     - port: 80
       targetPort: 80

src/build.js

@@ -24,6 +24,7 @@ const CONTACT_EMAIL = process.env.CONTACT_EMAIL || 'signalledger@jopdorp.nl';
 const PARENT_COMPANY = process.env.PARENT_COMPANY || 'Jopdorp';
 const ADSENSE_CLIENT = process.env.ADSENSE_CLIENT || 'ca-pub-1269854634225826';
 const ADSENSE_SLOT = process.env.ADSENSE_SLOT || '7019613848';
+const ADSENSE_IN_ARTICLE_SLOT = process.env.ADSENSE_IN_ARTICLE_SLOT || '9095112841';
 const IMAGE_MODEL = process.env.OPENAI_IMAGE_MODEL || 'gpt-image-1';
 const GENERATED_IMAGE_LIMIT = Number(process.env.GENERATED_IMAGE_LIMIT || 3);
 const parser = new Parser({ timeout: 15000 });
@@ -531,10 +532,43 @@ async function ensureArticleImage(item, imageMeta, client, { tryRemote = true, a
   item.hasAvif = false;
 }
 
-function renderShell({ title, description, pathName = '/', bodyClass = '', content, generatedAt }) {
+function renderArticleJsonLd(item) {
+  const json = {
+    '@context': 'https://schema.org',
+    '@type': 'NewsArticle',
+    headline: item.title,
+    description: item.summary || item.dek || SITE_TAGLINE,
+    image: item.imagePath ? `${SITE_URL}${item.imagePath}` : undefined,
+    url: `${SITE_URL}${item.articlePath}`,
+    datePublished: item.published,
+    dateModified: item.published,
+    author: {
+      '@type': 'Organization',
+      name: SITE_NAME,
+      url: SITE_URL,
+    },
+    publisher: {
+      '@type': 'Organization',
+      name: SITE_NAME,
+      url: SITE_URL,
+      logo: {
+        '@type': 'ImageObject',
+        url: `${SITE_URL}/favicon.svg`,
+      },
+    },
+    mainEntityOfPage: {
+      '@type': 'WebPage',
+      '@id': `${SITE_URL}${item.articlePath}`,
+    },
+  };
+  if (!json.image) delete json.image;
+  return `<script type="application/ld+json">${JSON.stringify(json)}</script>`;
+}
+
+function renderShell({ title, description, pathName = '/', bodyClass = '', content, generatedAt, jsonLd = '' }) {
   const canonical = `${SITE_URL}${pathName}`;
   const adsenseHead = ADSENSE_CLIENT ? `<meta name="google-adsense-account" content="${escapeHtml(ADSENSE_CLIENT)}">\n  <script async src="https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js?client=${escapeHtml(ADSENSE_CLIENT)}" crossorigin="anonymous"></script>` : '';
-  return `<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>${escapeHtml(title)}</title><meta name="description" content="${escapeHtml(description)}"><meta property="og:title" content="${escapeHtml(title)}"><meta property="og:description" content="${escapeHtml(description)}"><meta property="og:type" content="article"><meta property="og:url" content="${escapeHtml(canonical)}"><link rel="canonical" href="${escapeHtml(canonical)}">${adsenseHead}<link rel="icon" type="image/svg+xml" href="/favicon.svg"><link rel="alternate" type="application/rss+xml" title="${escapeHtml(SITE_NAME)} RSS" href="${SITE_URL}/feed.xml"><link rel="stylesheet" href="/styles.css"></head><body class="${escapeHtml(bodyClass)}"><div class="page"><header class="site-header"><div class="brand-bar"><a class="wordmark" href="/">${escapeHtml(SITE_NAME)}</a><p class="tagline">${escapeHtml(SITE_TAGLINE)}</p></div><nav class="main-nav"><a href="/">Home</a><a href="/archive/">Archive</a><a href="/about/">About</a><a href="/feed.xml">RSS</a></nav><div class="edition-bar">Updated ${escapeHtml(formatDate(generatedAt))}</div></header>${content}<footer class="site-footer"><div class="footer-grid"><div><h4>${escapeHtml(SITE_NAME)}</h4><p>${escapeHtml(SITE_TAGLINE)}</p></div><div><h4>Editorial model</h4><p>Signal Ledger publishes concise magazine-style briefs, context, and viewpoint grounded in clearly attributed source reporting.</p></div><div><h4>Contact</h4><p><a href="mailto:${escapeHtml(CONTACT_EMAIL)}">${escapeHtml(CONTACT_EMAIL)}</a></p><p>${escapeHtml(SITE_NAME)} is a subsidiary of ${escapeHtml(PARENT_COMPANY)}.</p></div></div></footer></div></body></html>`;
+  return `<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>${escapeHtml(title)}</title><meta name="description" content="${escapeHtml(description)}"><meta property="og:title" content="${escapeHtml(title)}"><meta property="og:description" content="${escapeHtml(description)}"><meta property="og:type" content="article"><meta property="og:url" content="${escapeHtml(canonical)}"><link rel="canonical" href="${escapeHtml(canonical)}">${jsonLd}${adsenseHead}<link rel="icon" type="image/svg+xml" href="/favicon.svg"><link rel="alternate" type="application/rss+xml" title="${escapeHtml(SITE_NAME)} RSS" href="${SITE_URL}/feed.xml"><link rel="stylesheet" href="/styles.css"></head><body class="${escapeHtml(bodyClass)}"><div class="page"><header class="site-header"><div class="brand-bar"><a class="wordmark" href="/">${escapeHtml(SITE_NAME)}</a><p class="tagline">${escapeHtml(SITE_TAGLINE)}</p></div><nav class="main-nav"><a href="/">Home</a><a href="/archive/">Archive</a><a href="/about/">About</a><a href="/feed.xml">RSS</a></nav><div class="edition-bar">Updated ${escapeHtml(formatDate(generatedAt))}</div></header>${content}<footer class="site-footer"><div class="footer-grid"><div><h4>${escapeHtml(SITE_NAME)}</h4><p>${escapeHtml(SITE_TAGLINE)}</p></div><div><h4>Editorial model</h4><p>Signal Ledger publishes concise magazine-style briefs, context, and viewpoint grounded in clearly attributed source reporting.</p></div><div><h4>Contact</h4><p><a href="mailto:${escapeHtml(CONTACT_EMAIL)}">${escapeHtml(CONTACT_EMAIL)}</a></p><p>${escapeHtml(SITE_NAME)} is a subsidiary of ${escapeHtml(PARENT_COMPANY)}.</p></div></div></footer></div></body></html>`;
 }
 
 function renderMainAdUnit(label = 'main page ads') {
@@ -542,6 +576,11 @@ function renderMainAdUnit(label = 'main page ads') {
   return `<aside class="ad-unit"><div class="ad-unit-label">${escapeHtml(label)}</div><script async src="https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js?client=${escapeHtml(ADSENSE_CLIENT)}" crossorigin="anonymous"></script><ins class="adsbygoogle" style="display:block" data-ad-client="${escapeHtml(ADSENSE_CLIENT)}" data-ad-slot="${escapeHtml(ADSENSE_SLOT)}" data-ad-format="auto" data-full-width-responsive="true"></ins><script>(adsbygoogle = window.adsbygoogle || []).push({});</script></aside>`;
 }
 
+function renderInArticleAdUnit(label = 'in-article ads') {
+  if (!ADSENSE_CLIENT || !ADSENSE_IN_ARTICLE_SLOT) return '';
+  return `<aside class="ad-unit"><div class="ad-unit-label">${escapeHtml(label)}</div><script async src="https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js?client=${escapeHtml(ADSENSE_CLIENT)}" crossorigin="anonymous"></script><ins class="adsbygoogle" style="display:block" data-ad-client="${escapeHtml(ADSENSE_CLIENT)}" data-ad-slot="${escapeHtml(ADSENSE_IN_ARTICLE_SLOT)}" data-ad-format="auto" data-full-width-responsive="true"></ins><script>(adsbygoogle = window.adsbygoogle || []).push({});</script></aside>`;
+}
+
 function renderImageCredit(item) {
   if (!item.imageCredit && !item.imageLicense) return '';
   const parts = [item.imageCredit, item.imageLicense].filter(Boolean);
@@ -589,8 +628,8 @@ function renderHome(items, groups, generatedAt) {
 function renderArticle(item, related, generatedAt) {
   const sourceHref = item.link && item.link !== '#' ? `<p><a href="${escapeHtml(item.link)}" rel="noreferrer noopener">Read the original reporting</a></p>` : '';
   const relatedInline = related.length ? `<section><h2>Where this fits in Signal Ledger</h2><p>This story sits alongside related Signal Ledger coverage that helps frame the broader pattern.</p><ul class="article-links">${related.slice(0, 3).map((other) => `<li><a href="${escapeHtml(other.articlePath)}">${escapeHtml(other.title)}</a></li>`).join('')}</ul></section>` : '';
-  const content = `<main class="article-layout"><article class="article">${renderPicture(item, { className: 'article-image' })}${renderImageCredit(item)}<div class="article-kicker">${escapeHtml(item.category)}</div><h1>${escapeHtml(item.title)}</h1>${item.dek ? `<p class="article-dek">${escapeHtml(item.dek)}</p>` : ''}<div class="article-meta">Published <time datetime="${escapeHtml(item.published)}">${escapeHtml(formatDate(item.published))}</time></div><section><h2>${escapeHtml(item.summaryLabel)}</h2><p>${escapeHtml(item.summary)}</p></section><section><h2>${escapeHtml(item.stakesLabel)}</h2><p>${escapeHtml(item.whyItMatters)}</p></section><section><h2>${escapeHtml(item.contextLabel)}</h2><p>${escapeHtml(item.context)}</p></section>${relatedInline}<section><h2>${escapeHtml(item.viewLabel)}</h2><p>${escapeHtml(item.viewpoint)}</p></section><section class="source-note"><h2>Source note</h2><p>${escapeHtml(item.sourceNote)}</p>${sourceHref}</section></article><aside class="sidebar">${renderMainAdUnit('article page ads')}<div class="sidebar-box"><h3>Related coverage</h3>${related.map((other) => `<p><a href="${escapeHtml(other.articlePath)}">${escapeHtml(other.title)}</a></p>`).join('') || '<p>No related items yet.</p>'}</div><div class="sidebar-box"><h3>Magazine index</h3><p><a href="/archive/">Browse archive</a></p><p><a href="/sections/${escapeHtml(item.sectionSlug)}/">More ${escapeHtml(item.category)}</a></p></div></aside></main>`;
+  const content = `<main class="article-layout"><article class="article">${renderPicture(item, { className: 'article-image' })}${renderImageCredit(item)}<div class="article-kicker">${escapeHtml(item.category)}</div><h1>${escapeHtml(item.title)}</h1>${item.dek ? `<p class="article-dek">${escapeHtml(item.dek)}</p>` : ''}<div class="article-meta">Published <time datetime="${escapeHtml(item.published)}">${escapeHtml(formatDate(item.published))}</time></div><section><h2>${escapeHtml(item.summaryLabel)}</h2><p>${escapeHtml(item.summary)}</p></section>${renderInArticleAdUnit()}<section><h2>${escapeHtml(item.stakesLabel)}</h2><p>${escapeHtml(item.whyItMatters)}</p></section><section><h2>${escapeHtml(item.contextLabel)}</h2><p>${escapeHtml(item.context)}</p></section>${relatedInline}<section><h2>${escapeHtml(item.viewLabel)}</h2><p>${escapeHtml(item.viewpoint)}</p></section><section class="source-note"><h2>Source note</h2><p>${escapeHtml(item.sourceNote)}</p>${sourceHref}</section></article><aside class="sidebar"><div class="sidebar-box"><h3>Related coverage</h3>${related.map((other) => `<p><a href="${escapeHtml(other.articlePath)}">${escapeHtml(other.title)}</a></p>`).join('') || '<p>No related items yet.</p>'}</div><div class="sidebar-box"><h3>Magazine index</h3><p><a href="/archive/">Browse archive</a></p><p><a href="/sections/${escapeHtml(item.sectionSlug)}/">More ${escapeHtml(item.category)}</a></p></div></aside></main>`;
-  return renderShell({ title: `${item.title} — ${SITE_NAME}`, description: item.summary || SITE_TAGLINE, pathName: item.articlePath, bodyClass: 'article-page', content, generatedAt });
+  return renderShell({ title: `${item.title} — ${SITE_NAME}`, description: item.summary || SITE_TAGLINE, pathName: item.articlePath, bodyClass: 'article-page', content, generatedAt, jsonLd: renderArticleJsonLd(item) });
 }
 
 function renderSectionPage(group, generatedAt) {
@@ -622,6 +661,38 @@ async function writeCss() {
   await fs.writeFile(path.join(OUT_DIR, 'styles.css'), css);
 }
 
+async function cleanOldArticles(currentSlugs) {
+  const slugSet = new Set(currentSlugs);
+  try {
+    const entries = await fs.readdir(ARTICLES_DIR, { withFileTypes: true });
+    for (const entry of entries) {
+      if (entry.isDirectory() && !slugSet.has(entry.name)) {
+        const dirPath = path.join(ARTICLES_DIR, entry.name);
+        await fs.rm(dirPath, { recursive: true, force: true });
+        console.log('Cleaned old article directory:', entry.name);
+      }
+    }
+  } catch (error) {
+    console.error('Error cleaning old articles:', error?.message || error);
+  }
+}
+
+async function cleanOldSections(currentSectionSlugs) {
+  const slugSet = new Set(currentSectionSlugs);
+  try {
+    const entries = await fs.readdir(SECTIONS_DIR, { withFileTypes: true });
+    for (const entry of entries) {
+      if (entry.isDirectory() && !slugSet.has(entry.name)) {
+        const dirPath = path.join(SECTIONS_DIR, entry.name);
+        await fs.rm(dirPath, { recursive: true, force: true });
+        console.log('Cleaned old section directory:', entry.name);
+      }
+    }
+  } catch (error) {
+    console.error('Error cleaning old sections:', error?.message || error);
+  }
+}
+
 async function main() {
   await fs.mkdir(OUT_DIR, { recursive: true });
   await fs.mkdir(ARTICLES_DIR, { recursive: true });
@@ -703,12 +774,16 @@ async function main() {
     await fs.writeFile(path.join(dir, 'index.html'), renderArticle(item, related, generatedAt));
   }
 
+  // Clean up old article and section directories that are no longer in the corpus
+  await cleanOldArticles(corpus.map((item) => item.slug));
+  await cleanOldSections(allGroups.map((g) => g.slug));
+
   await fs.writeFile(STATE_FILE, JSON.stringify({ generatedAt: generatedAt.toISOString(), articles: corpus }, null, 2));
   await fs.writeFile(path.join(OUT_DIR, 'feed.json'), JSON.stringify({ siteName: SITE_NAME, siteTagline: SITE_TAGLINE, generatedAt: generatedAt.toISOString(), items: homepageItems }, null, 2));
   await fs.writeFile(path.join(OUT_DIR, 'feed.xml'), renderRss(homepageItems, generatedAt));
   await fs.writeFile(path.join(OUT_DIR, 'sitemap.xml'), renderSitemap(corpus, allGroups));
   await fs.writeFile(path.join(OUT_DIR, 'robots.txt'), `User-agent: *\nAllow: /\nSitemap: ${SITE_URL}/sitemap.xml\n`);
-  await fs.writeFile(path.join(OUT_DIR, 'ads.txt'), `google.com, pub-1269854634225826, DIRECT, f08c47fec0942fa0\n`);
+  await fs.writeFile(path.join(OUT_DIR, 'ads.txt'), `google.com, ${ADSENSE_CLIENT.replace('ca-pub-', 'pub-')}, DIRECT, f08c47fec0942fa0\n`);
   console.log(`Built ${corpus.length} stories into ${OUT_DIR}`);
 }