render

[render]: draw video frames and result arrays onto the panels of a video.

config.render says what a video is (ADR-0031); this module draws it frame by frame, with Pillow and VideoWriter, handed the resolved table, the arrays through a read callable, the video readers and the rig, and knows nothing of runs. A panel shows one view, or a whole video with every view it carries drawn in the video's own pixels. Content fits its panel by uniform scale, centered, and every 2D coordinate goes through the same map, so a cropped picture and the points over it agree. A frame is a NumPy canvas: OpenCV resizes the footage into it and draws the vector marks straight onto it, anti-aliased against what is under them, and the text is FreeType's through Pillow, rendered once per string (_Marks, ADR-0143). The treadmill ball is drawn from what [ball] writes: its outline, and the trail the animal has walked over it (ADR-0147).

Layer dataclass

One thing to draw: a draw kind over one data reference, with its style.

The renderer's record of a layers entry (Layer says which keys each kind takes); every field but draw has a default.

Source code in src/deeperfly/render.py

@dataclass(frozen=True)
class Layer:
    """One thing to draw: a draw kind over one data reference, with its style.

    The renderer's record of a layers entry ([Layer][deeperfly.config.render.Layer]
    says which keys each kind takes); every field but `draw` has a default.
    """

    draw: str
    data: str | None = None
    panel: tuple[str, ...] = ()
    select: tuple[str, ...] = ()
    color: str | None = None
    colors: Mapping[str, str] = field(default_factory=dict)
    edge_color: str | None = None
    radius: float | None = None
    width: float | None = None
    dash: tuple[float, float] | None = None
    outline: float | None = None
    alpha: float | str | None = None
    alpha_range: tuple[float, float] | None = None
    identity: str = "hue"
    hide: tuple[str, ...] = ()
    window_frames: int = 0
    text: str | None = None
    size: float | None = None
    anchor: str | None = None

RenderReport dataclass

What one video came to.

Attributes:

Name	Type	Description
path	Path	The MP4 written.
n_frames	int	Frames drawn.
fps	float	The file's frame rate.
hold	int	Output frames each drawn frame is held for, so the file carries n_frames * hold frames.
size	tuple[int, int]	The (height, width) written.

Source code in src/deeperfly/render.py

@dataclass(frozen=True)
class RenderReport:
    """What one video came to.

    Attributes:
        path: The MP4 written.
        n_frames: Frames drawn.
        fps: The file's frame rate.
        hold: Output frames each drawn frame is held for, so the file carries
            `n_frames * hold` frames.
        size: The `(height, width)` written.
    """

    path: Path
    n_frames: int
    fps: float
    hold: int
    size: tuple[int, int]

SyntheticView dataclass

A view no camera recorded: an orbit placement to draw 3D data from.

distance, focal and image_size may be left to the renderer, which fits them to the data.

Source code in src/deeperfly/render.py

@dataclass(frozen=True)
class SyntheticView:
    """A view no camera recorded: an orbit placement to draw 3D data from.

    `distance`, `focal` and `image_size` may be left to the renderer, which fits them
    to the data.
    """

    azimuth_deg: float = 0.0
    elevation_deg: float = 0.0
    roll_deg: float = 0.0
    look_at: tuple[float, float, float] = (0.0, 0.0, 0.0)
    distance: float | None = None
    focal: float | None = None
    image_size: tuple[int, int] | None = None

alpha_map

alpha_map(values, alpha_range: tuple[float, float] | None)

An array's values as opacities in [0, 1], NaN transparent.

As they are without a range; with [lo, hi], lo maps to 0 and hi to 1, so [20, 0] fades what is large.

Source code in src/deeperfly/render.py

def alpha_map(values, alpha_range: tuple[float, float] | None):
    """An array's values as opacities in `[0, 1]`, NaN transparent.

    As they are without a range; with `[lo, hi]`, `lo` maps to 0 and `hi` to 1, so
    `[20, 0]` fades what is large.
    """
    a = np.asarray(values, dtype=float)
    if alpha_range is not None:
        lo, hi = float(alpha_range[0]), float(alpha_range[1])
        a = (a - lo) / (hi - lo) if hi != lo else np.where(a >= hi, 1.0, 0.0)
    return np.clip(np.nan_to_num(a, nan=0.0), 0.0, 1.0)

frames_in_range

frames_in_range(
    frame_range: Sequence[int], frames: ndarray
) -> np.ndarray

The frames a half-open [start, stop] or [start, stop, step] range keeps.

frames are the frames asked for; an empty range keeps them all, and a step counts from start.

Source code in src/deeperfly/render.py

def frames_in_range(frame_range: Sequence[int], frames: np.ndarray) -> np.ndarray:
    """The frames a half-open `[start, stop]` or `[start, stop, step]` range keeps.

    `frames` are the frames asked for; an empty range keeps them all, and a step
    counts from `start`.
    """
    frames = np.asarray(frames, dtype=np.int64)
    if not frame_range:
        return frames
    start, stop, *rest = (int(v) for v in frame_range)
    step = rest[0] if rest else 1
    keep = (frames >= start) & (frames < stop) & ((frames - start) % step == 0)
    return frames[keep]

layout

layout(
    grid: Sequence[Sequence[str]], cell: tuple[int, int]
) -> dict[str, Box]

Each grid name's canvas rectangle for cells of (height, width).

Source code in src/deeperfly/render.py

def layout(grid: Sequence[Sequence[str]], cell: tuple[int, int]) -> dict[str, Box]:
    """Each grid name's canvas rectangle for cells of `(height, width)`."""
    ch, cw = int(cell[0]), int(cell[1])
    found: dict[str, list[tuple[int, int]]] = {}
    for r, row in enumerate(grid):
        for c, name in enumerate(row):
            if name:
                found.setdefault(name, []).append((r, c))
    out: dict[str, Box] = {}
    for name, cells in found.items():
        r0, r1 = min(r for r, _ in cells), max(r for r, _ in cells)
        c0, c1 = min(c for _, c in cells), max(c for _, c in cells)
        out[name] = (c0 * cw, r0 * ch, (c1 - c0 + 1) * cw, (r1 - r0 + 1) * ch)
    return out

render_video

render_video(
    video: Mapping[str, Any],
    *,
    out: str | Path,
    project: str,
    videos: Mapping[str, FrameReader],
    frames: Sequence[int] | ndarray,
    fps: float | None,
    rig: Rig,
    skeleton: Skeleton,
    views: Mapping[str, SyntheticView],
    read: Callable[[str], ndarray],
    names: Callable[[str], tuple[str, ...] | None],
    axes: Callable[[str], tuple] | None = None,
    mesh: ModelMesh | None = None,
    layout: Layout | None = None,
    log: Callable[[str], None] = print,
) -> RenderReport

Draw one video and write it as an MP4.

video is one resolved [render.outputs.<name>] table with [render]'s defaults filled in, and frames the increasing frame indices to draw, one output frame each: the caller applies the table's own range (frames_in_range). videos maps each video to its reader, decoded once per frame whatever its panels, and layout says which views each video carries (omitted, one view per video, of its name). A grid name is a view of rig, a video of videos, which shows its whole frame with every view it carries drawn in, or a synthetic view of views. project is the name a {project} caption shows. A view this project lacks renders as an empty panel with a notice. fps defaults to FALLBACK_FPS. read resolves a data reference, names a reference's named axis and axes its declared axes (which tell a per-animal array apart). mesh is the body model's overlay mesh, which the caller loads from the pack [ik] names. The video is drawn at its cell a panel and, with a max_size, scaled down to fit it before it is written.

Raises:

Type	Description
ValueError	On a trace over an array without channel names, or a mesh layer with no mesh passed in.

Source code in src/deeperfly/render.py

def render_video(
    video: Mapping[str, Any],
    *,
    out: str | Path,
    project: str,
    videos: Mapping[str, FrameReader],
    frames: Sequence[int] | np.ndarray,
    fps: float | None,
    rig: Rig,
    skeleton: Skeleton,
    views: Mapping[str, SyntheticView],
    read: Callable[[str], np.ndarray],
    names: Callable[[str], tuple[str, ...] | None],
    axes: Callable[[str], tuple] | None = None,
    mesh: "ModelMesh | None" = None,
    layout: Layout | None = None,
    log: Callable[[str], None] = print,
) -> RenderReport:
    """Draw one video and write it as an MP4.

    `video` is one resolved `[render.outputs.<name>]` table with `[render]`'s defaults
    filled in, and `frames` the increasing frame indices to draw, one output frame
    each: the caller applies the table's own range
    ([frames_in_range][deeperfly.render.frames_in_range]). `videos` maps each video
    to its reader, decoded once per frame whatever its panels, and `layout` says which
    views each video carries (omitted, one view per video, of its name). A grid name
    is a view of `rig`, a video of `videos`, which shows its whole frame with every
    view it carries drawn in, or a synthetic view of `views`. `project` is the name a
    `{project}` caption shows. A view this project lacks renders as an empty panel
    with a notice. `fps` defaults to `FALLBACK_FPS`. `read` resolves a data
    reference, `names` a reference's named axis and `axes` its declared axes (which
    tell a per-animal array apart). `mesh` is the body model's overlay mesh, which
    the caller loads from the pack `[ik]` names. The video is drawn at its `cell` a
    panel and, with a `max_size`, scaled down to fit it before it is written.

    Raises:
        ValueError: On a trace over an array without channel names, or a `mesh`
            layer with no mesh passed in.
    """
    import cv2

    out = Path(out)
    layers: tuple[Layer, ...] = tuple(video["layers"])
    if mesh is None and any(layer.draw == "mesh" for layer in layers):
        raise ValueError("render: a 'mesh' layer needs the body model's overlay mesh")
    indices = np.unique(np.asarray(frames, dtype=np.int64))
    if indices.size == 0:
        raise ValueError("render: no frame to draw")
    # The motion's pace follows the closest two drawn frames: a stepped range plays
    # at the step, a set of stretches at the recording's own rate.
    step = int(np.diff(indices).min()) if indices.size > 1 else 1
    source_fps = float(fps) if fps else FALLBACK_FPS
    if not fps:
        log(f"    the videos carry no frame rate; assuming {FALLBACK_FPS:g} fps")
    played_fps = source_fps * float(video["speed"]) / step
    hold, out_fps = _hold(played_fps, float(video["fps"]))

    arrays = _Arrays(read, names, axes)
    posed = None if mesh is None else _Posed(mesh, arrays)
    carried = layout if layout is not None else Layout()
    panels = _panels(video, videos, rig, views, layers, arrays, indices, carried)
    cell = video["cell"]
    height = len(video["grid"]) * int(cell[0])
    width = len(video["grid"][0]) * int(cell[1])
    size = _written_size((height, width), video.get("max_size") or (0, 0))
    background = _rgb(video["background"])
    vars_ = {"project": project, "fps": source_fps}

    # One decode per video, however many of its views have a panel; only footage
    # panels name one.
    shown = list(dict.fromkeys(p.video for p in panels if p.video is not None))
    runs = _arithmetic_runs(indices)
    streams = [_stream(videos[name], runs) for name in shown]
    decoded: Iterable = (
        prefetch(zip(*streams, strict=False), depth=2)
        if streams
        else ((),) * len(indices)
    )
    crf = video.get("crf")
    options = None if crf is None else {"crf": str(crf)}
    written = 0
    canvas = np.empty((height, width, 3), dtype=np.uint8)
    with VideoWriter(out, out_fps, codec=video["codec"], options=options) as writer:
        for t, raw in zip(indices.tolist(), decoded, strict=False):
            canvas[:] = background
            marks = _Marks(canvas)
            current = dict(zip(shown, raw, strict=True))
            for layer in layers:
                targets = _targets(layer, panels)
                for panel in targets:
                    _draw_layer(
                        canvas,
                        marks,
                        layer,
                        panel,
                        t,
                        arrays,
                        current,
                        skeleton,
                        vars_,
                        background,
                        posed,
                    )
            frame = canvas
            if size != (height, width):
                frame = cv2.resize(canvas, size[::-1], interpolation=cv2.INTER_AREA)
            # The writer copies the frame, so the canvas is reused.
            for _ in range(hold):
                writer.write_frame(frame)
            written += 1
    drawn = "" if size == (height, width) else f", drawn at {width}x{height}"
    log(
        f"    {out.name}: {written} frames at {out_fps:g} fps, "
        f"{size[1]}x{size[0]}{drawn}"
    )
    if hold > 1:
        # Holding raises the file's rate but not the motion's, so say it: the fix is
        # a faster `speed`, and a video nobody meant to be slow is worth noticing.
        log(
            f"      each frame held {hold}x -- {played_fps:g} fps of motion in a "
            f"{out_fps:g} fps file; raise speed to smooth it"
        )
    return RenderReport(out, written, out_fps, hold, size)