Switch face detection to CNN model with parallel batch processing

- faces.py: use model="cnn" (more accurate, better at angles/small faces/poor
  lighting) instead of HOG; model comment explains the trade-off clearly
- faces.py: accept multiple image paths; process with ThreadPoolExecutor
  (dlib releases GIL during C++ inference → genuine thread parallelism);
  output JSON dict {path: [faces]} for batch calls
- update.rb: batch_detect_faces() collects all unprocessed images per
  directory and calls faces.py once per directory rather than once per image,
  avoiding repeated model load overhead
- update.rb: FACES_WORKERS read from config.yml faces.workers (default 4;
  set to 20 in this install's config.yml on a 64-core Xeon)
- update.rb: process_dir() now takes idx/total and prints [N/total] prefix
  on every Scanning/Skipping line for progress monitoring

To monitor a long run:
  nohup ruby /opt/albumen/scripts/update.rb > /tmp/faces_update.log 2>&1 &
  tail -f /tmp/faces_update.log

Resume/restart is fully safe: sentinel files are only written after
atomic_write_json, so an aborted directory reruns cleanly from scratch.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

2 files changed, 93 insertions, 49 deletions

diff --git a/scripts/faces.py b/scripts/faces.py
index d072376..2390a68 100644
--- a/scripts/faces.py
+++ b/scripts/faces.py
@@ -1,50 +1,77 @@
 #!/usr/bin/env python3
 """
-Detect faces in an image and return their bounding boxes and 128-D encodings.
+Detect faces in one or more images and return bounding boxes and 128-D encodings.
 
-Usage: python3 faces.py <image_path>
+Usage: python3 faces.py [--workers N] <image_path> [<image_path> ...]
 
-Stdout: JSON array — one object per face:
-  [{"box": [top, right, bottom, left], "encoding": [128 floats]}, ...]
+Output: JSON dict mapping each input path to its result array.
+  {"/path/img.jpg": [{"box": [top, right, bottom, left], "encoding": [128 floats]}, ...], ...}
 
-Returns "[]" when no faces are found or the image cannot be opened.
-Errors are written to stderr; stdout is always valid JSON.
+A null value for a path means detection failed (file unreadable, corrupt, etc.);
+update.rb leaves that file's 'faces' field as null and retries on the next run.
+An empty array [] means the image was processed successfully but no faces were found.
+
+Model note
+----------
+Uses the CNN model (model="cnn"), which is substantially more accurate than the
+HOG model, especially for:
+  - Faces at angles (up to ~45° profile)
+  - Small faces in group photos
+  - Faces in non-ideal lighting
+
+Trade-off: CNN is ~10-30x slower than HOG on CPU. Parallelism via --workers
+compensates on multi-core machines. dlib releases the Python GIL during C++
+inference, so threads achieve genuine concurrency.
+
+To switch to the faster but less accurate HOG model, change model="cnn" to
+model="hog" in the detect_one() function below.
 """
 import sys
 import json
+import argparse
+from concurrent.futures import ThreadPoolExecutor
 
+try:
+    import face_recognition
+    _FR_AVAILABLE = True
+except ImportError as _e:
+    print(f"face_recognition not available: {_e}", file=sys.stderr)
+    _FR_AVAILABLE = False
 
-def main():
-    if len(sys.argv) < 2:
-        print("[]")
-        return
-
-    path = sys.argv[1]
-    try:
-        import face_recognition
-    except ImportError as e:
-        print(f"face_recognition not available: {e}", file=sys.stderr)
-        print("[]")
-        return
 
+def detect_one(path):
+    """Returns list of face dicts, or None on error."""
     try:
         img = face_recognition.load_image_file(path)
+        locations = face_recognition.face_locations(img, model="cnn")
+        encodings = face_recognition.face_encodings(img, locations)
+        return [{"box": list(loc), "encoding": enc.tolist()}
+                for loc, enc in zip(locations, encodings)]
     except Exception as e:
-        print(f"Could not load {path}: {e}", file=sys.stderr)
-        print("[]")
+        print(f"  {path}: {e}", file=sys.stderr)
+        return None
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__,
+                                     formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("images", nargs="*", help="Image paths to process")
+    parser.add_argument("--workers", type=int, default=4,
+                        help="Parallel threads (default: 4; set to nproc for full utilisation)")
+    args = parser.parse_args()
+
+    if not _FR_AVAILABLE or not args.images:
+        print("{}")
         return
 
-    try:
-        locations = face_recognition.face_locations(img, model="hog")
-        encodings = face_recognition.face_encodings(img, locations)
-        result = [
-            {"box": list(loc), "encoding": enc.tolist()}
-            for loc, enc in zip(locations, encodings)
-        ]
-        print(json.dumps(result))
-    except Exception as e:
-        print(f"Detection error for {path}: {e}", file=sys.stderr)
-        print("[]")
+    if len(args.images) == 1 or args.workers <= 1:
+        results = {p: detect_one(p) for p in args.images}
+    else:
+        with ThreadPoolExecutor(max_workers=args.workers) as pool:
+            face_lists = list(pool.map(detect_one, args.images))
+        results = dict(zip(args.images, face_lists))
+
+    print(json.dumps(results))
 
 
 if __name__ == "__main__":
diff --git a/scripts/update.rb b/scripts/update.rb
index d6effe5..1a00ddf 100644
--- a/scripts/update.rb
+++ b/scripts/update.rb
@@ -35,8 +35,9 @@ MEDIA_EXTS     = (IMAGE_EXTS + VIDEO_EXTS + AUDIO_EXTS).freeze
 TRANSCODE_EXTS = %w[avi mkv mov].freeze  # not universally browser-playable; convert to MP4
 SENTINEL_FILE  = '.albumen_scanned'.freeze
 
-_cfg         = File.exist?(CONFIG_PATH) ? YAML.load_file(CONFIG_PATH, symbolize_names: true) : {}
+_cfg          = File.exist?(CONFIG_PATH) ? YAML.load_file(CONFIG_PATH, symbolize_names: true) : {}
 FACES_ENABLED = (_cfg.dig(:faces, :enabled) == true).freeze
+FACES_WORKERS = (_cfg.dig(:faces, :workers) || 4).freeze
 VENV_PYTHON   = File.expand_path('../venv/bin/python3', __dir__).freeze
 FACES_SCRIPT  = File.expand_path('faces.py', __dir__).freeze
 
@@ -45,19 +46,20 @@ FORCE_UPDATE = !!(ARGV.delete('--force') || ARGV[0])
 
 # ── Directory processing ───────────────────────────────────────────────────────
 
-def process_dir(dir)
-  rel = dir.delete_prefix(MEDIA_ROOT).delete_prefix('/')
-  label = rel.empty? ? '(root)' : rel
+def process_dir(dir, idx, total)
+  rel    = dir.delete_prefix(MEDIA_ROOT).delete_prefix('/')
+  label  = rel.empty? ? '(root)' : rel
+  prefix = "[#{idx}/#{total}]"
 
   unless FORCE_UPDATE
     sentinel = File.join(dir, SENTINEL_FILE)
     if File.exist?(sentinel) && File.mtime(sentinel) >= File.mtime(dir)
-      puts "Skipping #{label} (unchanged)"
+      puts "#{prefix} Skipping #{label} (unchanged)"
       return
     end
   end
 
-  puts "Scanning #{label}"
+  puts "#{prefix} Scanning #{label}"
 
   json_path = File.join(dir, 'album.json')
   data = load_json(json_path)
@@ -125,6 +127,8 @@ def process_dir(dir)
     generate_thumb_if_needed(full, rel, name, ext)
   end
 
+  batch_detect_faces(dir, current, data) if FACES_ENABLED
+
   atomic_write_json(json_path, data)
   FileUtils.touch(File.join(dir, SENTINEL_FILE))
 end
@@ -178,23 +182,35 @@ def enrich_image(full, name, meta)
     end
   end
 
-  enrich_faces(full, name, meta)
 end
 
-def enrich_faces(full, name, meta)
-  return unless FACES_ENABLED
-  return unless meta['faces'].nil?  # already processed ([] means "processed, none found")
+def batch_detect_faces(dir, names, data)
   return unless File.exist?(VENV_PYTHON) && File.exist?(FACES_SCRIPT)
 
+  unprocessed = names.select do |name|
+    IMAGE_EXTS.include?(File.extname(name).downcase.delete_prefix('.')) &&
+      (data['files'][name] || {})['faces'].nil?
+  end
+  return if unprocessed.empty?
+
+  puts "  Detecting faces in #{unprocessed.length} image(s) (#{FACES_WORKERS} workers)…"
+  paths = unprocessed.map { |n| File.join(dir, n) }
+  cmd   = [VENV_PYTHON, FACES_SCRIPT, '--workers', FACES_WORKERS.to_s] + paths
+
   begin
-    out = IO.popen([VENV_PYTHON, FACES_SCRIPT, full], err: '/dev/null', &:read).strip
-    faces = JSON.parse(out.empty? ? '[]' : out)
-    if faces.is_a?(Array)
-      meta['faces'] = faces
+    out     = IO.popen(cmd, err: '/dev/null', &:read).strip
+    results = JSON.parse(out.empty? ? '{}' : out)
+    raise 'expected Hash' unless results.is_a?(Hash)
+
+    results.each do |path, faces|
+      name = File.basename(path)
+      next unless data['files'].key?(name)
+      next if faces.nil?   # error on this file — leave faces: null to retry
+      data['files'][name]['faces'] = faces
       puts "  #{name}: #{faces.length} face(s)" unless faces.empty?
     end
   rescue StandardError => e
-    warn "  #{name}: face detection error — #{e.message}"
+    warn "  Face detection batch error — #{e.message}"
   end
 end
 
@@ -325,8 +341,9 @@ if Process.uid == 0
 end
 
 # Walk the tree: process each directory (depth-first, parent before children)
-dirs = [start]
-dirs += Dir.glob("#{start}/**/*/").sort
-dirs.uniq.each { |d| process_dir(d) }
+dirs  = [start] + Dir.glob("#{start}/**/*/").sort
+dirs  = dirs.uniq
+total = dirs.size
+dirs.each_with_index { |d, i| process_dir(d, i + 1, total) }
 
 puts 'Done.'