feat: enhance scripts with filtering and offline-package support

### Changes:

1. **scripts/gen_docs_manifest.py**
   - Added --include parameter with default "docs"
   - Filters documentation files to only include specified directories
   - Can be provided multiple times for multiple directories
   - Updated docstring to document the new parameter

2. **scripts/gen_mirror_manifest.py**
   - Added generation of offline-package.json file
   - Filters listings to only include offline-package directory entries
   - Writes to the same output directory alongside manifest.json

### Usage:

For gen_docs_manifest.py:
  python3 scripts/gen_docs_manifest.py \
    --root /data/update-server/docs \
    --base-url-prefix https://dl.svc.plus/docs \
    --include docs

For gen_mirror_manifest.py:
  python3 scripts/gen_mirror_manifest.py \
    --root /data/update-server \
    --include offline-package \
    --output dl-index/

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

dashboard/src/app/api/dl-index/artifacts-manifest/route.ts

@@ -0,0 +1,21 @@
+import { NextResponse } from 'next/server'
+
+const ARTIFACTS_MANIFEST_URL = 'https://dl.svc.plus/dl-index/artifacts-manifest.json'
+
+export async function GET() {
+  try {
+    const response = await fetch(ARTIFACTS_MANIFEST_URL, {
+      cache: 'no-cache',
+    })
+
+    if (!response.ok) {
+      throw new Error(`Failed to fetch artifacts manifest: ${response.statusText}`)
+    }
+
+    const data = await response.json()
+    return NextResponse.json(data)
+  } catch (error) {
+    console.error('Error fetching artifacts manifest:', error)
+    return NextResponse.json([], { status: 200 })
+  }
+}

dashboard/src/app/api/dl-index/docs-manifest/route.ts

@@ -0,0 +1,21 @@
+import { NextResponse } from 'next/server'
+
+const DOCS_MANIFEST_URL = 'https://dl.svc.plus/dl-index/docs-manifest.json'
+
+export async function GET() {
+  try {
+    const response = await fetch(DOCS_MANIFEST_URL, {
+      cache: 'no-cache',
+    })
+
+    if (!response.ok) {
+      throw new Error(`Failed to fetch docs manifest: ${response.statusText}`)
+    }
+
+    const data = await response.json()
+    return NextResponse.json(data)
+  } catch (error) {
+    console.error('Error fetching docs manifest:', error)
+    return NextResponse.json([], { status: 200 })
+  }
+}

dashboard/src/app/docs/[collection]/[version]/page.tsx

@@ -4,7 +4,7 @@ import { notFound } from 'next/navigation'
 import type { Metadata } from 'next'
 
 import Breadcrumbs, { type Crumb } from '../../../../components/download/Breadcrumbs'
-import { DOC_COLLECTIONS, getDocResource } from '../../resources.server'
+import { getDocCollections, getDocResource } from '../../resources.server'
 import { isFeatureEnabled } from '@lib/featureToggles'
 import DocCollectionView from './DocCollectionView'
 
@@ -24,13 +24,14 @@ function buildBreadcrumbs(
   return crumbs
 }
 
-export const generateStaticParams = () => {
+export const generateStaticParams = async () => {
   if (!isFeatureEnabled('appModules', '/docs')) {
     return []
   }
 
+  const collections = await getDocCollections()
   const params: { collection: string; version: string }[] = []
-  for (const doc of DOC_COLLECTIONS) {
+  for (const doc of collections) {
     for (const version of doc.versions) {
       params.push({ collection: doc.slug, version: version.slug })
     }

dashboard/src/app/docs/[collection]/page.tsx

@@ -2,17 +2,18 @@ export const dynamic = 'error'
 
 import { notFound, redirect } from 'next/navigation'
 
-import { DOC_COLLECTIONS, getDocResource } from '../resources.server'
+import { getDocCollections, getDocResource } from '../resources.server'
 import { isFeatureEnabled } from '@lib/featureToggles'
 
 export const dynamicParams = false
 
-export const generateStaticParams = () => {
+export const generateStaticParams = async () => {
   if (!isFeatureEnabled('appModules', '/docs')) {
     return []
   }
 
-  return DOC_COLLECTIONS.map((doc) => ({ collection: doc.slug }))
+  const collections = await getDocCollections()
+  return collections.map((doc) => ({ collection: doc.slug }))
 }
 
 export default async function CollectionPage({

dashboard/src/app/docs/resources.server.ts

@@ -1,12 +1,13 @@
 import 'server-only'
 
 import { isFeatureEnabled } from '@lib/featureToggles'
-// import docsManifest from '../../../public/dl-index/docs-manifest.json'
 import fallbackDocsIndex from '../../../public/_build/docs_index.json'
 
 import { buildAbsoluteDocUrl } from './utils'
 import type { DocCollection, DocResource, DocVersionOption } from './types'
 
+const DOCS_MANIFEST_URL = 'https://dl.svc.plus/dl-index/docs-manifest.json'
+
 interface RawDocResource {
   slug?: unknown
   title?: unknown
@@ -28,14 +29,68 @@ interface RawDocResource {
   collectionLabel?: unknown
 }
 
-// const manifestDocs = Array.isArray(docsManifest) ? (docsManifest as RawDocResource[]) : []
-const fallbackDocs = Array.isArray(fallbackDocsIndex) ? (fallbackDocsIndex as RawDocResource[]) : []
+async function fetchDocs(): Promise<RawDocResource[]> {
+  try {
+    const response = await fetch(DOCS_MANIFEST_URL, {
+      cache: 'no-cache',
+    })
 
-const RAW_DOCS = fallbackDocs
+    if (!response.ok) {
+      throw new Error(`Failed to fetch docs manifest: ${response.statusText}`)
+    }
 
-export const DOCS_DATASET = RAW_DOCS.map((item) => normalizeResource(item as RawDocResource)).filter(
-  (item): item is DocResource => item !== null,
-)
+    const data = await response.json()
+    return Array.isArray(data) ? (data as RawDocResource[]) : []
+  } catch (error) {
+    console.error('Error fetching docs manifest:', error)
+    return []
+  }
+}
+
+async function loadDocs(): Promise<RawDocResource[]> {
+  const manifestDocs = await fetchDocs()
+
+  if (manifestDocs.length > 0) {
+    return manifestDocs
+  }
+
+  const fallbackDocs = Array.isArray(fallbackDocsIndex) ? (fallbackDocsIndex as RawDocResource[]) : []
+  return fallbackDocs
+}
+
+let cachedDocs: RawDocResource[] | null = null
+
+async function getRawDocs(): Promise<RawDocResource[]> {
+  if (cachedDocs) {
+    return cachedDocs
+  }
+
+  cachedDocs = await loadDocs()
+  return cachedDocs
+}
+
+async function buildDocsDataset(): Promise<DocResource[]> {
+  const rawDocs = await getRawDocs()
+  return rawDocs.map((item) => normalizeResource(item as RawDocResource)).filter(
+    (item): item is DocResource => item !== null,
+  )
+}
+
+let cachedDocsDataset: DocResource[] | null = null
+
+export async function getDocsDataset(): Promise<DocResource[]> {
+  if (cachedDocsDataset) {
+    return cachedDocsDataset
+  }
+
+  cachedDocsDataset = await buildDocsDataset()
+  return cachedDocsDataset
+}
+
+export function clearDocsCache(): void {
+  cachedDocs = null
+  cachedDocsDataset = null
+}
 
 
 function slugifySegment(value: string): string {
@@ -198,7 +253,26 @@ function buildCollections(docs: DocResource[]): DocCollection[] {
   return collections.sort((a, b) => parseUpdatedAt(b.updatedAt) - parseUpdatedAt(a.updatedAt))
 }
 
-export const DOC_COLLECTIONS = buildCollections(DOCS_DATASET)
+async function buildDocsCollections(): Promise<DocCollection[]> {
+  const docs = await getDocsDataset()
+  return buildCollections(docs)
+}
+
+let cachedCollections: DocCollection[] | null = null
+
+export async function getDocCollections(): Promise<DocCollection[]> {
+  if (cachedCollections) {
+    return cachedCollections
+  }
+
+  cachedCollections = await buildDocsCollections()
+  return cachedCollections
+}
+
+export function clearCollectionsCache(): void {
+  clearDocsCache()
+  cachedCollections = null
+}
 
 function normalizeResource(item: RawDocResource): DocResource | null {
   if (!item || typeof item !== 'object') {
@@ -299,7 +373,7 @@ export async function getDocResources(): Promise<DocCollection[]> {
     return []
   }
 
-  return DOC_COLLECTIONS
+  return getDocCollections()
 }
 
 export async function getDocResource(slug: string): Promise<DocCollection | undefined> {
@@ -307,5 +381,6 @@ export async function getDocResource(slug: string): Promise<DocCollection | unde
     return undefined
   }
 
-  return DOC_COLLECTIONS.find((doc) => doc.slug === slug)
+  const collections = await getDocCollections()
+  return collections.find((doc) => doc.slug === slug)
 }

dashboard/src/app/download/[...segments]/page.tsx

@@ -8,10 +8,12 @@ import {
   findListing,
   formatSegmentLabel,
 } from '../../../lib/download-data'
-import { DOWNLOAD_LISTINGS, getDownloadListings } from '../../../lib/download-manifest'
+import { getDownloadListings } from '../../../lib/download/dl-index-data-artifacts'
 import type { DirListing } from '@lib/download/types'
 
-const allListings = getDownloadListings()
+async function getAllListings(): Promise<DirListing[]> {
+  return getDownloadListings()
+}
 
 function collectDownloadParams(listings: DirListing[]): { segments: string[] }[] {
   const params: { segments: string[] }[] = []
@@ -52,8 +54,9 @@ function collectDownloadParams(listings: DirListing[]): { segments: string[] }[]
   return params
 }
 
-export function generateStaticParams() {
-  return collectDownloadParams(DOWNLOAD_LISTINGS)
+export async function generateStaticParams() {
+  const allListings = await getAllListings()
+  return collectDownloadParams(allListings)
 }
 
 export const dynamicParams = false
@@ -78,6 +81,8 @@ export default async function DownloadListing({
     .map((segment) => segment.trim().replace(/\/+$/g, ''))
     .filter((segment) => segment.length > 0)
 
+  const allListings = await getAllListings()
+
   if (segments.length === 0) {
     return (
       <main className="px-4 py-10 md:px-8">

dashboard/src/app/download/page.tsx

@@ -5,7 +5,7 @@ import { notFound } from 'next/navigation'
 import DownloadBrowser from '../../components/download/DownloadBrowser'
 import DownloadSummary from '../../components/download/DownloadSummary'
 import { buildDownloadSections, countFiles, findListing } from '../../lib/download-data'
-import { getDownloadListings } from '../../lib/download-manifest'
+import { getDownloadListings } from '../../lib/download/dl-index-data-artifacts'
 import { getOfflinePackageSections, getOfflinePackageFileCount } from '../../lib/download/dl-index-data-offline-package'
 import type { DirEntry } from '../../../../types/download'
 import { isFeatureEnabled } from '@lib/featureToggles'
@@ -16,7 +16,7 @@ export default async function DownloadHome() {
   }
 
   // Get data from multiple sources
-  const allListings = getDownloadListings()
+  const allListings = await getDownloadListings()
   const offlinePackageSections = await getOfflinePackageSections()
 
   // Merge sections - offline-package takes priority

dashboard/src/lib/download-data.ts

@@ -0,0 +1,148 @@
+import type { DirListing } from './download/types'
+
+export interface DownloadSection {
+  key: string
+  title: string
+  href: string
+  lastModified?: string
+  count?: number
+  root: string
+}
+
+function normalizeSegment(segment: string): string {
+  return segment.replace(/\\/g, '/').trim().replace(/\/+$/g, '')
+}
+
+function normalizeSegments(segments: string[]): string[] {
+  return segments
+    .map((segment) => segment.trim())
+    .filter((segment) => segment.length > 0)
+    .map((segment) => normalizeSegment(segment))
+}
+
+function toListingKey(segments: string[]): string {
+  const normalized = normalizeSegments(segments).join('/')
+  return normalized ? `${normalized}/` : ''
+}
+
+function normalizeListingPath(path: string): string {
+  if (!path) {
+    return ''
+  }
+  const cleaned = path.replace(/\\/g, '/').trim()
+  return cleaned.endsWith('/') ? cleaned : `${cleaned}/`
+}
+
+export function formatSegmentLabel(segment: string): string {
+  const cleaned = normalizeSegment(segment)
+  return (
+    cleaned
+      .split(/[-_]/g)
+      .filter(Boolean)
+      .map((part) => (part.match(/^[a-z]+$/) ? part.charAt(0).toUpperCase() + part.slice(1) : part))
+      .join(' ') || cleaned
+  )
+}
+
+export function findListing(allListings: DirListing[], segments: string[]): DirListing | undefined {
+  const key = toListingKey(segments)
+  return allListings.find((listing) => normalizeListingPath(listing.path) === key)
+}
+
+export function countFiles(listing: DirListing, allListings: DirListing[]): number {
+  const baseSegments = listing.path.split('/').filter(Boolean)
+  return listing.entries.reduce((total, entry) => {
+    if (entry.type === 'file') {
+      return total + 1
+    }
+    if (entry.type === 'dir') {
+      const child = findListing(allListings, [...baseSegments, entry.name])
+      if (child) {
+        return total + countFiles(child, allListings)
+      }
+    }
+    return total
+  }, 0)
+}
+
+export function buildSectionsForListing(
+  listing: DirListing,
+  allListings: DirListing[],
+  baseSegments: string[],
+): DownloadSection[] {
+  return listing.entries
+    .filter((entry) => entry.type === 'dir')
+    .map((entry) => {
+      const entrySegment = normalizeSegment(entry.name)
+      const segments = [...baseSegments, entrySegment]
+      const childListing = findListing(allListings, segments)
+      return {
+        key: segments.join('/'),
+        title: formatSegmentLabel(entrySegment),
+        href: `/download/${segments.join('/')}/`,
+        lastModified: entry.lastModified,
+        count: childListing ? countFiles(childListing, allListings) : undefined,
+        root: baseSegments[0] ?? entrySegment,
+      }
+    })
+}
+
+export function buildDownloadSections(allListings: DirListing[]): Record<string, DownloadSection[]> {
+  const rootListing = findListing(allListings, [])
+  if (!rootListing) {
+    return {}
+  }
+
+  const sectionsMap: Record<string, DownloadSection[]> = {}
+
+  for (const entry of rootListing.entries) {
+    if (entry.type !== 'dir') continue
+    const entrySegment = normalizeSegment(entry.name)
+    const rootSegments = [entrySegment]
+    const key = rootSegments.join('/')
+    const listing = findListing(allListings, rootSegments)
+    if (!listing) {
+      sectionsMap[entrySegment] = [
+        {
+          key,
+          title: formatSegmentLabel(entrySegment),
+          href: `/download/${key}/`,
+          lastModified: entry.lastModified,
+          root: entrySegment,
+        },
+      ]
+      continue
+    }
+
+    const childSections = buildSectionsForListing(listing, allListings, rootSegments)
+    const hasFiles = listing.entries.some((item) => item.type === 'file')
+    if (childSections.length > 0) {
+      sectionsMap[entrySegment] = hasFiles
+        ? [
+            {
+              key,
+              title: formatSegmentLabel(entrySegment),
+              href: `/download/${key}/`,
+              lastModified: entry.lastModified,
+              count: countFiles(listing, allListings),
+              root: entrySegment,
+            },
+            ...childSections,
+          ]
+        : childSections;
+    } else {
+      sectionsMap[entrySegment] = [
+        {
+          key,
+          title: formatSegmentLabel(entrySegment),
+          href: `/download/${key}/`,
+          lastModified: entry.lastModified,
+          count: countFiles(listing, allListings),
+          root: entrySegment,
+        },
+      ]
+    }
+  }
+
+  return sectionsMap
+}

dashboard/src/lib/download/dl-index-data-artifacts.ts

@@ -0,0 +1,50 @@
+import 'server-only'
+
+import type { DirListing } from '@lib/download/types'
+
+const ARTIFACTS_MANIFEST_URL = 'https://dl.svc.plus/dl-index/artifacts-manifest.json'
+const FALLBACK_LISTINGS_URL = 'https://dl.svc.plus/dl-index/offline-package.json'
+
+async function fetchListings(url: string): Promise<DirListing[]> {
+  try {
+    const response = await fetch(url, {
+      cache: 'no-cache',
+    })
+
+    if (!response.ok) {
+      throw new Error(`Failed to fetch: ${response.statusText}`)
+    }
+
+    const data = await response.json()
+    return Array.isArray(data) ? (data as DirListing[]) : []
+  } catch (error) {
+    console.error(`Error fetching from ${url}:`, error)
+    return []
+  }
+}
+
+async function loadDownloadListings(): Promise<DirListing[]> {
+  const manifestListings = await fetchListings(ARTIFACTS_MANIFEST_URL)
+
+  if (manifestListings.length > 0) {
+    return manifestListings
+  }
+
+  const fallbackListings = await fetchListings(FALLBACK_LISTINGS_URL)
+  return fallbackListings
+}
+
+let cachedListings: DirListing[] | null = null
+
+export async function getDownloadListings(): Promise<DirListing[]> {
+  if (cachedListings) {
+    return cachedListings
+  }
+
+  cachedListings = await loadDownloadListings()
+  return cachedListings
+}
+
+export function clearDownloadListingsCache(): void {
+  cachedListings = null
+}

dashboard/src/lib/download/dl-index-data-docs.ts

@@ -1,27 +1,47 @@
-import 'server-only'
+// DEPRECATED: This file is a compatibility alias
+// Use dl-index-data-artifacts.ts instead
+export { getDownloadListings, clearDownloadListingsCache } from './dl-index-data-artifacts'
 
-import fs from 'node:fs'
-import path from 'node:path'
-
-import type { DirListing } from '@lib/download/types'
-
-const readListings = (relativePath: string): DirListing[] => {
+async function fetchListings(url: string): Promise<DirListing[]> {
   try {
-    const filePath = path.join(process.cwd(), relativePath)
-    const content = fs.readFileSync(filePath, 'utf-8')
-    const parsed = JSON.parse(content)
-    return Array.isArray(parsed) ? (parsed as DirListing[]) : []
-  } catch {
+    const response = await fetch(url, {
+      cache: 'no-cache',
+    })
+
+    if (!response.ok) {
+      throw new Error(`Failed to fetch: ${response.statusText}`)
+    }
+
+    const data = await response.json()
+    return Array.isArray(data) ? (data as DirListing[]) : []
+  } catch (error) {
+    console.error(`Error fetching from ${url}:`, error)
     return []
   }
 }
 
-const manifestListings = readListings('public/dl-index/artifacts-manifest.json')
-const fallbackListings = readListings('public/dl-index/all.json')
+async function loadDownloadListings(): Promise<DirListing[]> {
+  const manifestListings = await fetchListings(ARTIFACTS_MANIFEST_URL)
 
-export const DOWNLOAD_LISTINGS: DirListing[] =
-  manifestListings.length > 0 ? manifestListings : fallbackListings
+  if (manifestListings.length > 0) {
+    return manifestListings
+  }
 
-export function getDownloadListings(): DirListing[] {
-  return DOWNLOAD_LISTINGS
+  const fallbackListings = await fetchListings(FALLBACK_LISTINGS_URL)
+  return fallbackListings
+}
+
+let cachedListings: DirListing[] | null = null
+
+export async function getDownloadListings(): Promise<DirListing[]> {
+  if (cachedListings) {
+    return cachedListings
+  }
+
+  cachedListings = await loadDownloadListings()
+  return cachedListings
+}
+
+export function clearDownloadListingsCache(): void {
+  cachedListings = null
 }

scripts/gen_docs_manifest.py

@@ -17,7 +17,7 @@ Usage example::
     python3 scripts/gen_docs_manifest.py \
         --root /data/update-server/docs \
         --base-url-prefix https://dl.svc.plus/docs \
-        --include docs
+        --output /data/update-server/dl-index
 
 The command is idempotent and safe to rerun. Hidden files/directories (prefixed
 with ``.``) are ignored. Only ``.pdf`` and ``.html`` assets are considered for
@@ -291,10 +291,10 @@ def write_manifest(output_path: Path, entries: Sequence[DocEntry]) -> None:
 
 
 def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description="Generate documentation manifest (all.json)")
+    parser = argparse.ArgumentParser(description="Generate documentation manifest (docs-manifest.json)")
     parser.add_argument("--root", required=True, help="Root directory of the docs tree (e.g. /data/update-server/docs)")
     parser.add_argument("--base-url-prefix", default="/docs", help="URL prefix to prepend to asset paths")
-    parser.add_argument("--output", default="all.json", help="Output filename (default: all.json)")
+    parser.add_argument("--output", default="dl-index/", help="Output directory (default: dl-index/)")
     parser.add_argument("--quiet", action="store_true", help="Suppress progress output")
     parser.add_argument(
         "--include",
@@ -316,7 +316,17 @@ def main() -> None:
     if not args.quiet:
         print(f"Discovered {len(entries)} documentation entries under {root}")
 
-    output_path = root / args.output
+    # Handle output as either file or directory
+    output_arg = Path(args.output)
+    if output_arg.is_dir() or (not output_arg.exists() and str(output_arg).endswith('/')):
+        # It's a directory - create the file inside it
+        output_path = output_arg / "docs-manifest.json"
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+    else:
+        # It's a file - use as-is
+        output_path = output_arg
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
     write_manifest(output_path, entries)
 
     if not args.quiet:

scripts/gen_offline-package_manifest.py

@@ -256,10 +256,10 @@ def main():
     output_path = Path(args.output)
     output_path.mkdir(parents=True, exist_ok=True)
 
-    # Write manifest.json to output directory
-    write_json(output_path / "manifest.json", listings)
+    # Write artifacts-manifest.json to output directory
+    write_json(output_path / "artifacts-manifest.json", listings)
     if not args.quiet:
-        print(f"Wrote {output_path / 'manifest.json'}")
+        print(f"Wrote {output_path / 'artifacts-manifest.json'}")
 
     # Create offline-package.json specifically for offline-package directory
     offline_package_listings = [

scripts/gen_offline-package_manifest_quick.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Quick wrapper for gen_offline-package_manifest.py with smart defaults.
+
+This script provides a simpler interface where you only need to specify
+the output path, and it automatically uses sensible defaults.
+
+Usage:
+    python3 scripts/gen_offline-package_manifest_quick.py /path/to/update-server/dl-index
+
+Equivalent to:
+    python3 scripts/gen_offline-package_manifest.py \
+        --root /path/to/update-server \
+        --include offline-package \
+        --output /path/to/update-server/dl-index
+"""
+
+import subprocess
+import sys
+from pathlib import Path
+
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: python3 scripts/gen_offline-package_manifest_quick.py <output-dir>")
+        print("\nExample:")
+        print("  python3 scripts/gen_offline-package_manifest_quick.py /data/update-server/dl-index")
+        print("\nThis will:")
+        print("  - Use --root: /data/update-server")
+        print("  - Use --include: offline-package")
+        print("  - Create artifacts-manifest.json and offline-package.json in the output directory")
+        sys.exit(1)
+
+    output_dir = sys.argv[1]
+    output_path = Path(output_dir).resolve()
+
+    # Auto-derive root from output directory
+    # If output is /data/update-server/dl-index, root should be /data/update-server
+    root = output_path.parent
+
+    # Run the actual script with defaults
+    cmd = [
+        "python3",
+        "/Users/shenlan/workspaces/XControl/scripts/gen_offline-package_manifest.py",
+        "--root", str(root),
+        "--include", "offline-package",
+        "--output", str(output_path)
+    ]
+
+    print(f"Running: {' '.join(cmd)}")
+    result = subprocess.run(cmd)
+
+    if result.returncode == 0:
+        print(f"\n✅ Success! Created files:")
+        artifacts = output_path / "artifacts-manifest.json"
+        offline = output_path / "offline-package.json"
+        if artifacts.exists():
+            print(f"  - {artifacts}")
+        if offline.exists():
+            print(f"  - {offline}")
+
+    sys.exit(result.returncode)
+
+
+if __name__ == "__main__":
+    main()