From 73d6f8c9ac0177ca3a6587e6534592a545d44d67 Mon Sep 17 00:00:00 2001 From: Ken D'Ambrosio Date: Mon, 8 Jun 2026 17:34:18 +0000 Subject: Switch face detection to CNN model with parallel batch processing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - faces.py: use model="cnn" (more accurate, better at angles/small faces/poor lighting) instead of HOG; model comment explains the trade-off clearly - faces.py: accept multiple image paths; process with ThreadPoolExecutor (dlib releases GIL during C++ inference → genuine thread parallelism); output JSON dict {path: [faces]} for batch calls - update.rb: batch_detect_faces() collects all unprocessed images per directory and calls faces.py once per directory rather than once per image, avoiding repeated model load overhead - update.rb: FACES_WORKERS read from config.yml faces.workers (default 4; set to 20 in this install's config.yml on a 64-core Xeon) - update.rb: process_dir() now takes idx/total and prints [N/total] prefix on every Scanning/Skipping line for progress monitoring To monitor a long run: nohup ruby /opt/albumen/scripts/update.rb > /tmp/faces_update.log 2>&1 & tail -f /tmp/faces_update.log Resume/restart is fully safe: sentinel files are only written after atomic_write_json, so an aborted directory reruns cleanly from scratch. Co-Authored-By: Claude Sonnet 4.6 --- scripts/faces.py | 89 ++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 58 insertions(+), 31 deletions(-) (limited to 'scripts/faces.py') diff --git a/scripts/faces.py b/scripts/faces.py index d072376..2390a68 100644 --- a/scripts/faces.py +++ b/scripts/faces.py @@ -1,50 +1,77 @@ #!/usr/bin/env python3 """ -Detect faces in an image and return their bounding boxes and 128-D encodings. +Detect faces in one or more images and return bounding boxes and 128-D encodings. -Usage: python3 faces.py +Usage: python3 faces.py [--workers N] [ ...] -Stdout: JSON array — one object per face: - [{"box": [top, right, bottom, left], "encoding": [128 floats]}, ...] +Output: JSON dict mapping each input path to its result array. + {"/path/img.jpg": [{"box": [top, right, bottom, left], "encoding": [128 floats]}, ...], ...} -Returns "[]" when no faces are found or the image cannot be opened. -Errors are written to stderr; stdout is always valid JSON. +A null value for a path means detection failed (file unreadable, corrupt, etc.); +update.rb leaves that file's 'faces' field as null and retries on the next run. +An empty array [] means the image was processed successfully but no faces were found. + +Model note +---------- +Uses the CNN model (model="cnn"), which is substantially more accurate than the +HOG model, especially for: + - Faces at angles (up to ~45° profile) + - Small faces in group photos + - Faces in non-ideal lighting + +Trade-off: CNN is ~10-30x slower than HOG on CPU. Parallelism via --workers +compensates on multi-core machines. dlib releases the Python GIL during C++ +inference, so threads achieve genuine concurrency. + +To switch to the faster but less accurate HOG model, change model="cnn" to +model="hog" in the detect_one() function below. """ import sys import json +import argparse +from concurrent.futures import ThreadPoolExecutor +try: + import face_recognition + _FR_AVAILABLE = True +except ImportError as _e: + print(f"face_recognition not available: {_e}", file=sys.stderr) + _FR_AVAILABLE = False -def main(): - if len(sys.argv) < 2: - print("[]") - return - - path = sys.argv[1] - try: - import face_recognition - except ImportError as e: - print(f"face_recognition not available: {e}", file=sys.stderr) - print("[]") - return +def detect_one(path): + """Returns list of face dicts, or None on error.""" try: img = face_recognition.load_image_file(path) + locations = face_recognition.face_locations(img, model="cnn") + encodings = face_recognition.face_encodings(img, locations) + return [{"box": list(loc), "encoding": enc.tolist()} + for loc, enc in zip(locations, encodings)] except Exception as e: - print(f"Could not load {path}: {e}", file=sys.stderr) - print("[]") + print(f" {path}: {e}", file=sys.stderr) + return None + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("images", nargs="*", help="Image paths to process") + parser.add_argument("--workers", type=int, default=4, + help="Parallel threads (default: 4; set to nproc for full utilisation)") + args = parser.parse_args() + + if not _FR_AVAILABLE or not args.images: + print("{}") return - try: - locations = face_recognition.face_locations(img, model="hog") - encodings = face_recognition.face_encodings(img, locations) - result = [ - {"box": list(loc), "encoding": enc.tolist()} - for loc, enc in zip(locations, encodings) - ] - print(json.dumps(result)) - except Exception as e: - print(f"Detection error for {path}: {e}", file=sys.stderr) - print("[]") + if len(args.images) == 1 or args.workers <= 1: + results = {p: detect_one(p) for p in args.images} + else: + with ThreadPoolExecutor(max_workers=args.workers) as pool: + face_lists = list(pool.map(detect_one, args.images)) + results = dict(zip(args.images, face_lists)) + + print(json.dumps(results)) if __name__ == "__main__": -- cgit v1.2.3