feat: Leads Finder discovery + WordPress license validate + file storage
3 new PRD features: 1. Leads Finder discovery (PRD §6.2 Stage 1: Discover) - Real SerpAPI integration (Google Maps local search) - Real Tavily integration (web search API) - Real Firecrawl integration (search + scrape) - Returns company name, website, address, phone, description - API key required (503 with hint if not configured) - Rate limited, Zod validated, audit-logged - POST /api/aos/leads/discover 2. WordPress-compatible license validation (PRD §6.4) - POST /api/aos/licenses/validate - WordPress plugin updater response format - Domain auto-activation with max check - Offline grace period (14 day default, configurable) - Status checks: active/suspended/expired/revoked - Rate limited (30/min), Zod validated, audit-logged - Tested: active→valid✓, invalid→404, expired→403, suspended→403 3. Content-addressed file storage (PRD §5.5) - src/lib/storage.ts - SHA-256 hash addressing with sharded dirs (first 2 chars) - Atomic writes (temp file → rename) - Hash verified on read (tamper detection) - Deduplication (same content = same hash = one file) - Content type detection from magic numbers (PDF, PNG, JPEG, GIF, WebP, ZIP) - storeFile(), readFile(), fileExists(), deleteFile() 0 TS errors, ESLint clean.
This commit is contained in:
1 parent
e45b68bb8b
commit
2aa3496a90
4 files changed
+470
No files matched your search
@@ -0,0 +1,134 @@
|
||||
// Content-addressed file storage (PRD §5.5: "Storage: SHA-256 addressing,
|
||||
// hash verified on read, content type from bytes")
|
||||
//
|
||||
// Files are stored on a volume, addressed by their SHA-256 hash.
|
||||
// This ensures: deduplication, tamper detection, integrity verification.
|
||||
// Atomic writes prevent partial files.
|
||||
|
||||
import { createHash } from 'crypto'
|
||||
import { promises as fs } from 'fs'
|
||||
import path from 'path'
|
||||
|
||||
const STORAGE_DIR = process.env.STORAGE_DIR ?? path.join(process.cwd(), 'data', 'files')
|
||||
|
||||
// Ensure storage directory exists
|
||||
async function ensureDir() {
|
||||
await fs.mkdir(STORAGE_DIR, { recursive: true })
|
||||
}
|
||||
|
||||
// Compute SHA-256 hash of file content
|
||||
export function computeHash(content: Buffer | string): string {
|
||||
return createHash('sha256').update(content).digest('hex')
|
||||
}
|
||||
|
||||
// Store file content (atomic write via temp file + rename)
|
||||
// Returns the hash (which is also the storage path)
|
||||
export async function storeFile(
|
||||
content: Buffer,
|
||||
contentType: string = 'application/octet-stream'
|
||||
): Promise<{ hash: string; path: string; size: number; contentType: string }> {
|
||||
await ensureDir()
|
||||
const hash = computeHash(content)
|
||||
|
||||
// Sharded directory structure: first 2 chars as subdirectory
|
||||
const subdir = hash.slice(0, 2)
|
||||
const dir = path.join(STORAGE_DIR, subdir)
|
||||
await fs.mkdir(dir, { recursive: true })
|
||||
|
||||
const filePath = path.join(dir, hash)
|
||||
|
||||
// Check if file already exists (deduplication)
|
||||
try {
|
||||
await fs.access(filePath)
|
||||
// File exists — skip write (dedup)
|
||||
const stat = await fs.stat(filePath)
|
||||
return { hash, path: filePath, size: stat.size, contentType }
|
||||
} catch {
|
||||
// File doesn't exist — write atomically
|
||||
}
|
||||
|
||||
// Atomic write: temp file → rename
|
||||
const tempPath = filePath + '.tmp'
|
||||
await fs.writeFile(tempPath, content)
|
||||
await fs.rename(tempPath, filePath)
|
||||
|
||||
return { hash, path: filePath, size: content.length, contentType }
|
||||
}
|
||||
|
||||
// Read file content, verifying hash on read (tamper detection)
|
||||
export async function readFile(hash: string): Promise<{ content: Buffer; verified: boolean; size: number }> {
|
||||
const subdir = hash.slice(0, 2)
|
||||
const filePath = path.join(STORAGE_DIR, subdir, hash)
|
||||
|
||||
try {
|
||||
const content = await fs.readFile(filePath)
|
||||
|
||||
// Verify hash on read (PRD: "hash verified on read")
|
||||
const actualHash = computeHash(content)
|
||||
const verified = actualHash === hash
|
||||
|
||||
if (!verified) {
|
||||
console.warn(`File hash mismatch! Expected: ${hash}, got: ${actualHash}. File may be tampered.`)
|
||||
}
|
||||
|
||||
return { content, verified, size: content.length }
|
||||
} catch {
|
||||
throw new Error(`File not found: ${hash}`)
|
||||
}
|
||||
}
|
||||
|
||||
// Check if a file exists by hash
|
||||
export async function fileExists(hash: string): Promise<boolean> {
|
||||
const subdir = hash.slice(0, 2)
|
||||
const filePath = path.join(STORAGE_DIR, subdir, hash)
|
||||
try {
|
||||
await fs.access(filePath)
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Delete file by hash (only if not referenced elsewhere)
|
||||
export async function deleteFile(hash: string): Promise<boolean> {
|
||||
const subdir = hash.slice(0, 2)
|
||||
const filePath = path.join(STORAGE_DIR, subdir, hash)
|
||||
try {
|
||||
await fs.unlink(filePath)
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Detect content type from file bytes (magic numbers)
|
||||
export function detectContentType(bytes: Buffer): string {
|
||||
if (bytes.length < 4) return 'application/octet-stream'
|
||||
|
||||
// PDF: %PDF
|
||||
if (bytes[0] === 0x25 && bytes[1] === 0x50 && bytes[2] === 0x44 && bytes[3] === 0x46) {
|
||||
return 'application/pdf'
|
||||
}
|
||||
// PNG: 0x89504E47
|
||||
if (bytes[0] === 0x89 && bytes[1] === 0x50 && bytes[2] === 0x4E && bytes[3] === 0x47) {
|
||||
return 'image/png'
|
||||
}
|
||||
// JPEG: 0xFFD8
|
||||
if (bytes[0] === 0xFF && bytes[1] === 0xD8) {
|
||||
return 'image/jpeg'
|
||||
}
|
||||
// GIF: GIF8
|
||||
if (bytes[0] === 0x47 && bytes[1] === 0x49 && bytes[2] === 0x46) {
|
||||
return 'image/gif'
|
||||
}
|
||||
// WebP: RIFF....WEBP
|
||||
if (bytes[0] === 0x52 && bytes[1] === 0x49 && bytes[2] === 0x46 && bytes[3] === 0x46) {
|
||||
return 'image/webp'
|
||||
}
|
||||
// ZIP/docx/xlsx: PK
|
||||
if (bytes[0] === 0x50 && bytes[1] === 0x4B) {
|
||||
return 'application/zip'
|
||||
}
|
||||
|
||||
return 'application/octet-stream'
|
||||
}
|
||||
Reference in new issue
Block a user