LiteRT multimodal lab

LiteRT multimodal lab#

Companion notebook for the video Run multimodal AI on Android with Python and LiteRT.

Connect to PythonHere#

%load_ext pythonhere
%connect-there

Show download progress#

%%there kv
<DownloadProgress>:
    size_hint: .82, .82
    pos_hint: {'center_x': .5, 'center_y': .5}

    canvas:
        Color:
            rgba: .45, .52, .50, .16

        Line:
            width: dp(5)
            circle:
                self.center_x, \
                self.center_y, \
                min(self.width, self.height) * .34, \
                0, \
                360

        Color:
            rgba: .30, .40, .37, .95 if root.value > 0 else 0

        Line:
            width: dp(10) if root.value >= 100 else dp(9)
            circle:
                self.center_x, \
                self.center_y, \
                min(self.width, self.height) * .34, \
                105, \
                105 + 360 * min(max(root.value, 0), 100) / 100

    Label:
        text: "Ready" if root.value >= 100 else f"{root.value:3.0f}%"
        font_size: min(root.width, root.height) * (.11 if root.value >= 100 else .13)
        color: .25, .32, .30, .95
        center: root.center

%%there
from kivy.properties import NumericProperty
from kivy.uix.widget import Widget


class DownloadProgress(Widget):
    value = NumericProperty(0)

root.clear_widgets()
progress = DownloadProgress()
root.add_widget(progress)

Download the model#

%%there --worker
from pprint import pprint as pp
from kivy.clock import mainthread
from ml_here import download_hf_model, require_model

MODEL_REPO = "litert-community/SmolVLM2-500M"
MODEL_FILE = "SmolVLM2-500M.litertlm"


@mainthread
def update_progress(value):
    progress.value = value.percent or 0.0


download_hf_model(
    repo_id=MODEL_REPO,
    filename=MODEL_FILE,
    progress=update_progress,
)

pp(require_model(MODEL_REPO, MODEL_FILE))

'/storage/emulated/0/Android/data/me.herethere.pythonhere_dev/files/models/litert-community/SmolVLM2-500M/SmolVLM2-500M.litertlm'

Load the model#

%%there --worker
import litert_lm
engine = litert_lm.Engine(
    require_model(MODEL_REPO, MODEL_FILE),
    backend=litert_lm.Backend.GPU(),
    vision_backend=litert_lm.Backend.GPU(),
    enable_benchmark=True,
)
pp(engine)

Engine(model_path='/storage/emulated/0/Android/data/me.herethere.pythonhere_dev/files/models/litert-community/SmolVLM2-500M/SmolVLM2-500M.litertlm',
       backend=GPU(gpu_decode_steps_per_sync=None),
       max_num_tokens=None,
       max_num_images=None,
       cache_dir=None,
       vision_backend=GPU(gpu_decode_steps_per_sync=None),
       audio_backend=None,
       enable_speculative_decoding=None,
       lora_rank_config=None,
       activation_data_type=None,
       use_ringbuffers_local_attention=None,
       enable_ynnpack=False)

Generate text#

Create a streaming output view#

%%there kv
<ModelOutput>:
    size_hint: .88, .7
    pos_hint: {'center_x': .5, 'center_y': .5}

    Label:
        markup: True
        text: root.text
        font_size: dp(21)
        halign: "left"
        valign: "top"
        text_size: root.width, None
        pos: root.pos
        size: root.size

%%there
from kivy.properties import StringProperty
from kivy.uix.widget import Widget
from kivy.clock import mainthread


class ModelOutput(Widget):
    text = StringProperty("")
    committed = StringProperty("")
    newest = StringProperty("")

    @mainthread
    def append_chunk(self, chunk):
        self.committed += self.newest
        self.newest = chunk
        self.text = (
            f"[color=#E8EEF3]{self.committed}[/color]"
            f"[color=#FFD343][b]{self.newest}[/b][/color]"
        )

root.clear_widgets()
output = ModelOutput()
root.add_widget(output)

Ask a text question#

%%there --worker

sampler = litert_lm.SamplerConfig(
    temperature=0.7,
    top_p=0.9,
    top_k=40,
)

with engine.create_conversation(
    sampler_config=sampler,
    max_output_tokens=128,
) as conversation:
    for chunk in conversation.send_message_async(
        "What is on-device AI? Give a direct answer in one short paragraph."
    ):
        output.append_chunk(chunk["content"][0]["text"])
    info = conversation.get_benchmark_info()
    output.append_chunk("")

Show benchmark results#

%%there
print(
    "\n\tBenchmark info for the conversation\n"
    f"Init:    {info.init_time_in_second:.2f}s\n"
    f"TTFT:    {info.time_to_first_token_in_second:.2f}s\n"
    f"Prefill: {info.last_prefill_token_count} tokens "
    f"@ {info.last_prefill_tokens_per_second:.1f} tok/s\n"
    f"Decode:  {info.last_decode_token_count} tokens "
    f"@ {info.last_decode_tokens_per_second:.1f} tok/s"
)

	Benchmark info for the conversation
Init:    8.35s
TTFT:    1.24s
Prefill: 23 tokens @ 19.8 tok/s
Decode:  84 tokens @ 11.9 tok/s

Describe a camera photo#

Request camera permission#

%%there
from threading import Event
from android.permissions import Permission, request_permission

done = Event()
request_permission(Permission.CAMERA, lambda *_: done.set())
done.wait()

Preview and capture a photo#

%%there kv
AnchorLayout:
    anchor_x: "center"
    anchor_y: "center"

    Camera:
        id: camera
        play: True
        resolution: (640, 480)
        fit_mode: "contain"

        size_hint: None, None
        height: min(root.width, root.height * 3 / 4)
        width: self.height * 4 / 3

        canvas.before:
            PushMatrix
            Rotate:
                angle: -90
                origin: self.center

        canvas.after:
            PopMatrix

%%there
camera = root.ids.camera
camera.play = False

%%there
from os.path import abspath, getsize
from PIL import Image

texture = camera.texture
image = Image.frombytes("RGBA", texture.size, texture.pixels)

# Kivy textures use a bottom-left origin.
image = image.transpose(Image.Transpose.FLIP_TOP_BOTTOM)

# Match the rotation applied to the Android camera preview.
image = image.rotate(-90, expand=True).convert("RGB")

photo_path = abspath("from_camera.jpg")
image.save(photo_path, quality=90)

print(f"Photo: {photo_path}")
print(
    f"Image: {image.width} × {image.height}, "
    f"{image.mode}, {getsize(photo_path) / 1024:.1f} KiB"
)

Photo: /data/data/me.herethere.pythonhere_dev/files/upload/from_camera.jpg
Image: 480 × 640, RGB, 38.2 KiB

Generate a description#

%%there --worker
sampler = litert_lm.SamplerConfig(
    temperature=1.5,
    top_p=0.9,
    top_k=40,
)

with engine.create_conversation(
    sampler_config=sampler,
    max_output_tokens=96,
) as conversation:
    response = conversation.send_message(
        litert_lm.Contents.of(
            "What is it?",
            litert_lm.Content.ImageFile(absolute_path=photo_path),
        )
    )
    print(response["content"][0]["text"])

 A mug likely belongs to the maker of Android or Kinder Stills for Samsung of Ireland with the Kitter Still Designs company. Here's a visual guide to understanding what could possibly fit inside the mug. However, this is purely speculative:

A mug likely fits a teacup. The teacup has a handle, a handle inside it is difficult to determine and may not fit directly with the interior inside. It might fit into what looks similar to a m

Explore sampling variability#

Generate responses with different temperatures and seeds#

%%there --worker
prompt = (
    "Describe the main object in this image clearly and concisely "
    "in 2-3 sentences."
)

temperatures = [0.2, 0.5, 0.7, 1.0]
seeds = [1, 7, 42, 123, 999]

experiment_results = []

for temperature in temperatures:
    for seed in seeds:
        sampler = litert_lm.SamplerConfig(
            temperature=temperature,
            top_p=0.9,
            top_k=40,
            seed=seed,
        )

        with engine.create_conversation(
            sampler_config=sampler,
            max_output_tokens=96,
        ) as conversation:
            response = conversation.send_message(
                litert_lm.Contents.of(
                    prompt,
                    litert_lm.Content.ImageFile(absolute_path=photo_path),
                )
            )

            info = conversation.get_benchmark_info()

        experiment_results.append({
            "temperature": temperature,
            "seed": seed,
            "output": response["content"][0]["text"].strip(),
            "ttft_s": info.time_to_first_token_in_second,
            "prefill_tokens": info.last_prefill_token_count,
            "prefill_tok_s": info.last_prefill_tokens_per_second,
            "decode_tokens": info.last_decode_token_count,
            "decode_tok_s": info.last_decode_tokens_per_second,
        })

print(f"Done: {len(experiment_results)} runs")

Done: 20 runs

Compare the outputs#

import pandas as pd

experiment_results = %there get experiment_results

df = pd.DataFrame(experiment_results)

display(
    df[
        [
            "temperature",
            "seed",
            "decode_tokens",
            "output",
        ]
    ].style
    .format({
        "temperature": "{:.1f}",
    })
    .set_properties(
        subset=["output"],
        **{
            "white-space": "pre-wrap",
            "text-align": "left",
        },
    )
    .hide(axis="index")
)
temperature seed decode_tokens output
0.2 1 20 The main object in the image is a mug with a cartoon penguin and Android mascot.
0.2 7 20 The main object in the image is a mug with a cartoon penguin and Android robot design.
0.2 42 35 The main object in the image is a mug with a cartoon character design. The mug is clear and has a handle, which is placed on a white surface.
0.2 123 35 The main object in the image is a mug with a cartoon character design. The mug has a handle on the right side and is placed on a white surface.
0.2 999 21 The main object in the image is a mug with a cartoon penguin and Android character on it.
0.5 1 55 The main object in the image is a mug with a cartoon design. The mug features an Android robot and a penguin on it. The robot is green and has a yellow car on its head. The penguin is blue and has a yellow car on its head.
0.5 7 20 The main object in the image is a mug with a cartoon penguin and Android robot design.
0.5 42 67 The main object in the image is a mug with a cartoon character. The mug is clear and has a handle, which is located on the right side of the mug. The cartoon character is a robot and a penguin. The mug is placed on a white surface, which is likely a counter or a table.
0.5 123 19 The main object in the image is a mug with a cartoon penguin and robot design.
0.5 999 20 The main object in the image is a mug with a cartoon penguin and Android mascot.
0.7 1 39 The main object in the image is a mug with a cartoon drawing of a penguin, a smiling face, and a small blue robot. The mug is placed on a white surface.
0.7 7 46 The main object in the image is a mug with a cartoon design. The mug has a handle on the right side and a small cartoon character on it. The character is green with a blue bird on its head.
0.7 42 47 The main object in the image is a mug with a cartoon character drawing on it. The mug has a handle and is decorated with a cartoon character, including a blue penguin. The cartoon character is riding a skateboard.
0.7 123 19 The main object in the image is a mug with a cartoon penguin and robot drawing.
0.7 999 27 The main object in the image is a mug with a cartoon design. The mug features a cartoon robot and penguin characters.
1.0 1 53 The main object in the image is a mug with a design of an Android character with a bird and a penguin. The mug has a handle, a lid, and is decorated with a design of an Android character, a bird, and a penguin.
1.0 7 24 The main object in the image is a clear mug that has a cartoon penguin and android drawing on it.
1.0 42 18 The main object in the image is a mug featuring a cartoon character on it.
1.0 123 95 The main object in the image is a mug with an illustration of an android and penguin. The mug is set against a white background, which is commonly used in presentations to keep the focus on the subject without distractions. The illustration is centered, with a small, stylized mug icon in the top right corner, which suggests that the mug is not an actual mug, but rather an example to illustrate or enhance the mug's design or functionality.
1.0 999 37 The main object in the image is a mug with a picture of a robot and a penguin. The mug is made of glass and is colored in a light blue color.

Clean up#

%%there --worker
engine.close()
Contents