"""Batch convert photos to JXL, keep EXIF/ICC and file times, resumable.
- JPEG: long side -> 2400 px (LANCZOS), lossy (--distance/--effort); --in-process skips the cjxl process
- PNG/GIF/BMP/TIFF/PNM: full size, lossless; every result is decoded and compared pixel by pixel,
the original is copied instead if that check fails or JXL is not smaller
- duplicates (copies, re-saves, smaller versions; see dedupe.py) become .lnk shortcuts to one master
- other files: copied as-is up to COPY_MAX_MB
- video files: copied up to copy_video_size, larger ones recoded with ffmpeg (vcompress_cmd)
usage: python -I scripts/convert.py [--distance 2.6] [--effort 3] [--lossless-effort 7] [--workers 16]
[--limit N] [--copy-max-mb MB] [--in-process]
[--no-dedupe] [--dry-run]
"""
import argparse, ctypes, io, os, shutil, stat, subprocess, sys, time, traceback
from ctypes import wintypes
from collections import Counter
from multiprocessing import Pool
from pathlib import Path
import cv2
import numpy as np
from PIL import Image, ImageOps
sys.path.insert(0, str(Path(__file__).parent)) # python -I leaves the script folder off sys.path
import dedupe
SRC = Path(r"C:/Storage/Pictures")
DST = Path(r"C:/Backups/pic_jxld3e5")
COPY_MAX_MB = 1 # copy non-image files up to this size as-is; None = don't copy
LOSSY_EXTS = {".jpg", ".jpeg"}
LOSSLESS_EXTS = {".png", ".apng", ".gif", ".bmp", ".tif", ".tiff", ".ppm", ".pgm", ".pnm"}
VIDEO_EXTS = {".mkv", ".mp4", ".avi", ".mov"}
CJXL_READS = {".png", ".apng", ".gif", ".ppm", ".pgm", ".pnm"} # others go through Pillow -> PNG
PIL_SAFE_MODES = {"1", "L", "LA", "P", "PA", "RGB", "RGBA", "I;16"} # modes PNG stores without loss
LONG = 2400
ORIENTATION = 0x0112
# never converted, copied or reported (compared lower-case)
EXCLUDE_NAMES = {"picasa.ini", ".picasa.ini", "thumbs.db", "ehthumbs.db", "ehthumbs_vista.db",
"desktop.ini", ".ds_store", "iconcache.db"}
EXCLUDE_DIRS = {"$recycle.bin", "system volume information", ".picasaoriginals"}
always_copy = {"Sheet Music", "Wohnung Oberhaunstadt", "scans"}
copy_video_size = 50*1024*1024
vcompress_cmd = " -vf scale=1024:-1 -pix_fmt yuv420p -b:v 1024k -b:a 96k -profile:v main12 -movflags +faststart -vcodec libx265 "
def excluded(p):
if p.name.lower() in EXCLUDE_NAMES or any(d.lower() in EXCLUDE_DIRS for d in p.relative_to(SRC).parts[:-1]):
return True
return bool(p.stat().st_file_attributes & stat.FILE_ATTRIBUTE_SYSTEM)
def is_always_copy(p):
parts = p.relative_to(SRC).parts[:-1]
return any(d in always_copy or d.lower() in {x.lower() for x in always_copy} for d in parts)
_k32 = ctypes.WinDLL("kernel32", use_last_error=True)
_k32.CreateFileW.restype = wintypes.HANDLE
_k32.CreateFileW.argtypes = [wintypes.LPCWSTR, wintypes.DWORD, wintypes.DWORD, wintypes.LPVOID,
wintypes.DWORD, wintypes.DWORD, wintypes.HANDLE]
_k32.SetFileTime.argtypes = [wintypes.HANDLE] + [ctypes.POINTER(ctypes.c_ulonglong)] * 3
def copy_times(src, dst):
"""Give dst the created/accessed/modified times of src (os.utime can't set 'created' on Windows)."""
st = src.stat()
ft = [ctypes.c_ulonglong(ns // 100 + 116444736000000000) # ns since 1970 -> 100 ns since 1601
for ns in (st.st_ctime_ns, st.st_atime_ns, st.st_mtime_ns)] # st_ctime is 'created' on Windows
h = _k32.CreateFileW(str(dst), 0x100, 0, None, 3, 0x80, None) # FILE_WRITE_ATTRIBUTES, OPEN_EXISTING
if h == wintypes.HANDLE(-1).value:
raise ctypes.WinError(ctypes.get_last_error())
try:
if not _k32.SetFileTime(h, *(ctypes.byref(t) for t in ft)):
raise ctypes.WinError(ctypes.get_last_error())
finally:
_k32.CloseHandle(h)
def copy_file(src, dst):
dst.parent.mkdir(parents=True, exist_ok=True)
part = dst.with_name(dst.name + ".part")
shutil.copyfile(src, part)
os.replace(part, dst)
copy_times(src, dst)
def compress_video(src, dst, cmd_str):
dst.parent.mkdir(parents=True, exist_ok=True)
part = dst.with_name(dst.name + ".part")
cmd = ["ffmpeg", "-y", "-i", str(src)] + cmd_str.strip().split() + [str(part)]
r = subprocess.run(cmd, capture_output=True)
if r.returncode != 0:
part.unlink(missing_ok=True)
raise RuntimeError(r.stderr.decode(errors="replace").strip()[:300])
os.replace(part, dst)
copy_times(src, dst)
def prep(p):
im = Image.open(p)
exif = im.getexif()
icc = im.info.get("icc_profile")
w0, h0 = im.size
s = min(1.0, LONG / max(w0, h0)) # never upscale
tw, th = round(w0 * s), round(h0 * s)
if im.format == "JPEG":
im.draft("RGB", (tw, th)) # DCT-domain downscale (>= target), LANCZOS finishes
im = ImageOps.exif_transpose(im.convert("RGB"))
target = (tw, th) if (im.size[0] >= im.size[1]) == (tw >= th) else (th, tw) # transpose may swap axes
if im.size != target:
im = im.resize(target, Image.LANCZOS)
exif.pop(ORIENTATION, None) # pixels are already upright
return im, exif.tobytes() if len(exif) else None, icc
def quality_for_distance(d):
"""Inverse of libjxl's JxlEncoderDistanceFromQuality (the plugin only takes a quality)."""
if d <= 0:
return 100.0
if d <= 6.4: # quality 30..100 maps linearly to distance 6.4..0.1
return 100 - (d - 0.1) / 0.09
a, b, c = 53 / 3000, -23 / 20, 25 - d # below quality 30: d = a q^2 + b q + 25
return (-b - (b * b - 4 * a * c) ** 0.5) / (2 * a)
def encode_inproc(im, exif, opt):
from pillow_jxl import Encoder # only needed with --in-process
if exif:
tiff = exif[6:] if exif.startswith(b"Exif\x00\x00") else exif
exif = b"\0\0\0\0" + tiff # JXL Exif box starts with a 4-byte TIFF header offset; the plugin omits it
enc = Encoder(mode="RGB", lossless=False, quality=quality_for_distance(float(opt["distance"])),
decoding_speed=0, effort=int(opt["effort"]), use_container=True,
use_original_profile=False, num_threads=1)
return enc(im.tobytes(), im.width, im.height, jpeg_encode=False, exif=exif, jumb=None, xmp=None,
compress=True)
def lossy(src, dst, keep, opt):
im, exif, icc = prep(src)
dst.parent.mkdir(parents=True, exist_ok=True)
part = dst.with_name(dst.name + ".part")
if opt["in_process"] and not icc: # the plugin can't store an ICC profile -> those go through cjxl
part.write_bytes(encode_inproc(im, exif, opt))
else:
b = io.BytesIO()
kw = {k: v for k, v in (("exif", exif), ("icc_profile", icc)) if v}
im.save(b, "PNG", compress_level=0, **kw) # PNG so cjxl picks up EXIF/ICC
r = subprocess.run(["cjxl", "-", str(part), "-d", opt["distance"], "-e", opt["effort"], "--quiet",
"--num_threads=1"], input=b.getvalue(), capture_output=True)
if r.returncode:
part.unlink(missing_ok=True)
return src, dst, None, "fail", r.stderr.decode(errors="replace").strip()[:300]
os.replace(part, dst)
copy_times(src, dst)
return src, dst, dst.stat().st_size, "ok", ""
def as_rgba(a):
"""Bring a decoded array to H x W x 4 (gray -> 3 channels, missing alpha -> opaque) for comparison."""
if a.ndim == 2:
a = a[:, :, None]
if a.shape[2] in (1, 2):
a = np.concatenate([np.repeat(a[:, :, :1], 3, 2), a[:, :, 1:]], 2)
if a.shape[2] == 3:
a = np.concatenate([a, np.full(a.shape[:2] + (1,), np.iinfo(a.dtype).max, a.dtype)], 2)
return a
def same_pixels(src, decoded, animated):
if animated: # GIF/APNG: Pillow composites every frame; 8 bit only, which is all these formats hold
def frames(f):
im = Image.open(f)
out = []
for i in range(getattr(im, "n_frames", 1)):
im.seek(i)
out.append(np.asarray(im.convert("RGBA")))
return out
a, b = frames(src), frames(io.BytesIO(decoded))
return len(a) == len(b) and all(x.shape == y.shape and (x == y).all() for x, y in zip(a, b))
# still images: OpenCV keeps 16 bit and does not share code with the Pillow conversion path
a = cv2.imdecode(np.fromfile(src, np.uint8), cv2.IMREAD_UNCHANGED)
b = cv2.imdecode(np.frombuffer(decoded, np.uint8), cv2.IMREAD_UNCHANGED)
if a is None or b is None or a.dtype != b.dtype:
return False
a, b = as_rgba(a), as_rgba(b)
return a.shape == b.shape and bool((a == b).all())
def lossless(src, dst, keep, opt):
def keep_original(why):
copy_file(src, keep)
return src, keep, keep.stat().st_size, "keep", why
ext = src.suffix.lower()
try:
im = Image.open(src)
n_frames = getattr(im, "n_frames", 1)
if ext in (".tif", ".tiff") and n_frames > 1:
return keep_original(f"multi-page TIFF ({n_frames} pages)")
if ext in CJXL_READS:
inp, data = str(src), None
else:
if im.mode not in PIL_SAFE_MODES:
return keep_original(f"image mode {im.mode}")
b = io.BytesIO()
kw = {k: v for k, v in (("icc_profile", im.info.get("icc_profile")),
("exif", im.getexif().tobytes() if len(im.getexif()) else None)) if v}
im.save(b, "PNG", compress_level=0, **kw)
inp, data = "-", b.getvalue()
dst.parent.mkdir(parents=True, exist_ok=True)
part = dst.with_name(dst.name + ".part")
r = subprocess.run(["cjxl", inp, str(part), "-d", "0", "-e", opt["lossless_effort"], "--quiet",
"--num_threads=1"], input=data, capture_output=True)
if r.returncode:
part.unlink(missing_ok=True)
return keep_original("cjxl: " + r.stderr.decode(errors="replace").strip()[:200])
r = subprocess.run(["djxl", str(part), "-", "--output_format", "apng", "--quiet"], capture_output=True)
if r.returncode or not same_pixels(src, r.stdout, animated=n_frames > 1 or ext == ".gif"):
part.unlink(missing_ok=True)
return keep_original("pixel check failed")
if part.stat().st_size >= src.stat().st_size:
part.unlink(missing_ok=True)
return keep_original("JXL not smaller")
os.replace(part, dst)
copy_times(src, dst)
return src, dst, dst.stat().st_size, "ok", ""
except Exception:
dst.with_name(dst.name + ".part").unlink(missing_ok=True)
return keep_original(traceback.format_exception_only(*sys.exc_info()[:2])[-1].strip()[:200])
def work(a):
src, dst, keep, opt = a
try:
return (lossy if src.suffix.lower() in LOSSY_EXTS else lossless)(src, dst, keep, opt)
except Exception:
return src, dst, None, "fail", traceback.format_exception_only(*sys.exc_info()[:2])[-1].strip()[:300]
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--distance", default="3")
ap.add_argument("--effort", default="5")
ap.add_argument("--lossless-effort", default="7")
ap.add_argument("--in-process", action="store_true",
help="encode JPEGs with libjxl inside Python (pillow-jxl-plugin) instead of running cjxl; "
"photos with an ICC profile still use cjxl")
ap.add_argument("--workers", type=int, default=os.cpu_count())
ap.add_argument("--limit", type=int)
ap.add_argument("--no-dedupe", action="store_true", help="convert every image, no duplicate detection")
ap.add_argument("--dry-run", action="store_true", help="only print folder roles and planned shortcuts")
ap.add_argument("--copy-max-mb", type=float, metavar="MB", default=COPY_MAX_MB,
help=f"copy non-image files up to this size as-is (default: {COPY_MAX_MB}, negative: don't copy)")
a = ap.parse_args()
sys.stdout.reconfigure(errors="replace") # file names the console code page can't show must not crash the run
opt = {"distance": a.distance, "effort": a.effort, "lossless_effort": a.lossless_effort,
"in_process": a.in_process}
all_files = sorted(p for p in SRC.rglob("*") if p.is_file() and not excluded(p))
files = [p for p in all_files if not is_always_copy(p) and p.suffix.lower() in LOSSY_EXTS | LOSSLESS_EXTS]
stems = Counter((p.parent, p.stem.lower()) for p in files)
out = {} # source -> (dst .jxl, kept original, shortcut)
for p in files:
rel = p.relative_to(SRC)
# IMG.jpg + IMG.png in one folder would both become IMG.jxl -> name them IMG.jpg.jxl / IMG.png.jxl
dst = DST / (rel.parent / (rel.name + ".jxl") if stems[p.parent, p.stem.lower()] > 1 else rel.with_suffix(".jxl"))
out[p] = (dst, DST / rel, dst.with_name(dst.name + ".lnk"))
def done_already(p):
dst, keep, lnk = out[p]
return dst.exists() or lnk.exists() or (p.suffix.lower() in LOSSLESS_EXTS and keep.exists())
DST.mkdir(parents=True, exist_ok=True)
pool = Pool(a.workers)
links = {}
if not a.no_dedupe:
recs = dedupe.build_index(pool, files, DST / "_index.pkl")
links, folders = dedupe.plan(recs)
for f, (n, dup, coll) in sorted(folders.items()):
if dup:
print(f"{'collection' if coll else 'original '} {f} ({dup}/{n} images also elsewhere)", flush=True)
print(f"dedupe: {len(links)} of {len(files)} images are duplicates or smaller versions -> shortcuts", flush=True)
if a.dry_run:
for p, m in sorted(links.items()):
print(f"link {p} -> {m}", flush=True)
return
skipped = sum(done_already(p) for p in files)
jobs = [(p, *out[p][:2], opt) for p in files if p not in links and not done_already(p)][: a.limit]
link_jobs = [p for p in files if p in links and not done_already(p)]
copied = copy_done = ignored = video_compressed = video_copied = 0
for p in all_files:
ext = p.suffix.lower()
is_img = ext in LOSSY_EXTS | LOSSLESS_EXTS
is_vid = ext in VIDEO_EXTS
in_always = is_always_copy(p)
dst = DST / p.relative_to(SRC)
size = p.stat().st_size
if is_img and not in_always:
continue
if in_always:
if dst.exists() and dst.stat().st_size == size:
copy_done += 1
continue
copy_file(p, dst)
copied += 1
print(f"copy {p} -> {dst} (always_copy, {size/1e3:.0f} kB)", flush=True)
continue
if is_vid:
if size <= copy_video_size:
if dst.exists() and dst.stat().st_size == size:
copy_done += 1
continue
copy_file(p, dst)
video_copied += 1
print(f"copy {p} -> {dst} (video <= {copy_video_size/1e6:.1f}MB, {size/1e6:.1f} MB)", flush=True)
else:
if dst.exists():
video_compressed += 1
print(f"skip {p} (compressed video already exists)", flush=True)
continue
print(f"compress {p} -> {dst} ({size/1e6:.1f} MB > {copy_video_size/1e6:.1f}MB)", flush=True)
try:
compress_video(p, dst, vcompress_cmd)
video_compressed += 1
except Exception as e:
print(f"FAIL {p} ffmpeg: {e}", flush=True)
continue
if a.copy_max_mb is None or a.copy_max_mb < 0 or size > a.copy_max_mb * 1e6:
ignored += 1
why = "copy disabled" if a.copy_max_mb is None or a.copy_max_mb < 0 else f"{size/1e6:.1f} MB > {a.copy_max_mb:g} MB"
print(f"skip {p} ({why})", flush=True)
continue
if dst.exists() and dst.stat().st_size == size:
copy_done += 1
continue
copy_file(p, dst)
copied += 1
print(f"copy {p} -> {dst} ({size/1e3:.0f} kB)", flush=True)
print(f"{len(files)} images found, {skipped} already done, {len(jobs)} to convert, {len(link_jobs)} shortcuts to make; "
f"videos: {video_compressed} compressed, {video_copied} copied; "
f"other files: {copied} copied, {copy_done} already copied, {ignored} skipped", flush=True)
log = open(DST / "failures.log", "a", encoding="utf-8")
c = convert_all(pool, jobs, log)
# shortcuts point at whatever the master became; if the master failed, convert the duplicate itself
made, orphans = 0, []
attempted = {j[0] for j in jobs}
for p in link_jobs:
m = links[p]
target = next((t for t in out[m][:2] if t.exists()), None)
if target is None:
if m in attempted:
orphans.append((p, *out[p][:2], opt))
# else: master not converted yet (--limit), the shortcut follows in a later run
continue
try:
dedupe.make_shortcut(out[p][2], target)
copy_times(p, out[p][2])
made += 1
print(f"link {p} -> {target}", flush=True)
except Exception as e:
orphans.append((p, *out[p][:2], opt))
log.write(f"LINK\t{p}\t{e}\n"); log.flush()
if orphans:
print(f"{len(orphans)} duplicates have no usable master -> converting them instead", flush=True)
c2 = convert_all(pool, orphans, log)
c = {k: c[k] + c2[k] for k in c}
pool.close()
print(f"done={c['done']} kept_original={c['kept']} shortcuts={made} already_done={skipped} failed={c['fail']} "
f"(see {DST / 'failures.log'})")
def convert_all(pool, jobs, log):
t0 = time.perf_counter()
c = {"done": 0, "kept": 0, "fail": 0}
out_bytes = in_bytes = 0
# results arrive in the main process one at a time, so console lines never interleave
for src, dst, size, status, msg in pool.imap_unordered(work, jobs, chunksize=2):
n = sum(c.values()) + 1
if status == "fail":
c["fail"] += 1
log.write(f"FAIL\t{src}\t{msg}\n"); log.flush()
print(f"[{n}/{len(jobs)}] FAIL {src} {msg}", flush=True)
else:
out_bytes += size
in_bytes += src.stat().st_size
if status == "keep":
c["kept"] += 1
log.write(f"KEEP\t{src}\t{msg}\n"); log.flush()
print(f"[{n}/{len(jobs)}] keep {src} -> {dst} (original kept: {msg})", flush=True)
elif status == "ok":
c["done"] += 1
print(f"[{n}/{len(jobs)}] ok {src} -> {dst} ({size/1e3:.0f} kB)", flush=True)
if n % 200 == 0 or n == len(jobs):
el = time.perf_counter() - t0
rate = n / el
eta = (len(jobs) - n) / rate if rate else 0
print(f"{n}/{len(jobs)} {rate:.1f} img/s kept={c['kept']} fail={c['fail']} "
f"out={out_bytes/1e6:.0f}MB ({out_bytes/max(in_bytes,1):.3f} of src) "
f"elapsed={el/60:.1f}min eta={eta/3600:.2f}h", flush=True)
return c
if __name__ == "__main__":
main()