Audio Transcription & Speech

<dependency>
  <groupId>org.springframework.ai</groupId>
  <artifactId>spring-ai-starter-model-openai</artifactId>
</dependency>
<dependency>
  <groupId>org.springframework.ai</groupId>
  <artifactId>spring-ai-starter-model-elevenlabs</artifactId>
</dependency>

spring:
  ai:
    model:
      audio:
        transcription: openai
        speech: elevenlabs            # or openai
    openai:
      api-key: ${OPENAI_API_KEY}
      audio:
        transcription:
          model: gpt-transcribe
        speech:
          model: gpt-4o-mini-tts
          voice: alloy
    elevenlabs:
      api-key: ${ELEVENLABS_API_KEY}
      tts:
        model-id: eleven_multilingual_v2
        voice-id: ${ELEVENLABS_VOICE_ID}
docsassistant:
  voice:
    elevenlabs-voice-id: ${ELEVENLABS_VOICE_ID}

package com.example.docsassistant;

import org.springframework.ai.audio.transcription.TranscriptionModel;
import org.springframework.ai.openai.OpenAiAudioTranscriptionModel;
import org.springframework.ai.openai.OpenAiAudioTranscriptionOptions;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Service;
import reactor.core.publisher.Flux;

@Service
class TranscriptionService {

    private final TranscriptionModel transcriptionModel;
    private final OpenAiAudioTranscriptionOptions vocabulary;

    TranscriptionService(TranscriptionModel transcriptionModel) {
        this.transcriptionModel = transcriptionModel;
        // The builder of the per-request options fills in the class default model (whisper-1), which then wins
        // over the model you configured. Copying the model's own options first keeps the configured model.
        if (transcriptionModel instanceof OpenAiAudioTranscriptionModel openAi) {
            this.vocabulary = OpenAiAudioTranscriptionOptions.builder()
                    .from(openAi.getOptions())
                    .language("en")
                    .prompt("Antora, AsciiDoc, Asciidoctor, Spring AI, pgvector, MCP")
                    .build();
        }
        else {
            this.vocabulary = null;
        }
    }

    /** One call, the whole recording, the configured defaults. */
    String transcribe(Resource audio) {
        return this.transcriptionModel.transcribe(audio);
    }

    /** Language hint and a vocabulary prompt: fewer mistakes on the site's own product names. */
    String transcribeWithVocabulary(Resource audio) {
        if (this.vocabulary == null) {
            return this.transcriptionModel.transcribe(audio);
        }
        return this.transcriptionModel.transcribe(audio, this.vocabulary);
    }

    /** The text as the provider produces it. Needs a model that supports streaming; whisper-1 does not. */
    Flux<String> transcribeAsItArrives(Resource audio) {
        return this.transcriptionModel.streamTranscribe(audio)
                .filter(piece -> !piece.isEmpty());
    }
}

package com.example.docsassistant;

import org.springframework.ai.audio.tts.TextToSpeechModel;
import org.springframework.ai.audio.tts.TextToSpeechOptions;
import org.springframework.ai.audio.tts.TextToSpeechPrompt;
import org.springframework.ai.audio.tts.TextToSpeechResponse;
import org.springframework.ai.openai.OpenAiAudioSpeechOptions;
import org.springframework.stereotype.Service;
import reactor.core.publisher.Flux;

@Service
class SpeechService {

    private final TextToSpeechModel speechModel;

    SpeechService(TextToSpeechModel speechModel) {
        this.speechModel = speechModel;
    }

    /** The whole answer as one MP3 (OpenAI's defaults: gpt-4o-mini-tts, alloy, mp3). */
    byte[] speak(String text) {
        return this.speechModel.call(text);
    }

    /** Portable options: model, voice, format and speed exist on every provider; the values are the provider's. */
    byte[] speakPortable(String text) {
        TextToSpeechOptions options = TextToSpeechOptions.builder()
                .voice("nova")
                .format("opus")
                .speed(1.1)
                .build();
        TextToSpeechResponse response = this.speechModel.call(new TextToSpeechPrompt(text, options));
        return response.getResult().getOutput();
    }

    /** OpenAI-only option: instructions steer tone and pace on gpt-4o-mini-tts (ignored by tts-1). */
    byte[] speakCalmly(String text) {
        OpenAiAudioSpeechOptions options = OpenAiAudioSpeechOptions.builder()
                .model("gpt-4o-mini-tts")
                .voice(OpenAiAudioSpeechOptions.Voice.CORAL)
                .responseFormat(OpenAiAudioSpeechOptions.AudioResponseFormat.MP3)
                .instructions("Calm, friendly, clear pauses between sentences.")
                .build();
        return this.speechModel.call(new TextToSpeechPrompt(text, options)).getResult().getOutput();
    }

    /** Audio chunks as the provider produces them (at most 8 KB each on OpenAI). */
    Flux<byte[]> speakAsItArrives(String text) {
        return this.speechModel.stream(text);
    }
}

package com.example.docsassistant;

import org.springframework.ai.audio.transcription.TranscriptionModel;
import org.springframework.ai.audio.tts.TextToSpeechModel;
import org.springframework.ai.chat.client.ChatClient;
import org.springframework.ai.chat.memory.ChatMemory;
import org.springframework.core.io.Resource;
import org.springframework.security.core.Authentication;
import org.springframework.stereotype.Service;
import reactor.core.publisher.Flux;

/** Audio in, text through the ChatClient the rest of DocsAssistant uses, audio out. */
@Service
class VoiceAssistantService {

    private static final int MAX_SPOKEN_CHARS = 1500;

    private final TranscriptionModel transcription;
    private final TextToSpeechModel speech;
    private final ChatClient chatClient;
    private final ConversationIds conversationIds;

    VoiceAssistantService(TranscriptionModel transcription, TextToSpeechModel speech,
            ChatClient memoryChatClient, ConversationIds conversationIds) {
        this.transcription = transcription;
        this.speech = speech;
        this.chatClient = memoryChatClient;
        this.conversationIds = conversationIds;
    }

    record SpokenAnswer(String question, String answer, byte[] audio) {
    }

    SpokenAnswer ask(Authentication user, String conversation, Resource recording) {
        String question = this.transcription.transcribe(recording);
        String answer = answer(user, conversation, question);
        return new SpokenAnswer(question, answer, this.speech.call(SpokenText.forSpeech(answer, MAX_SPOKEN_CHARS)));
    }

    /** Same pipeline, but the audio is handed on chunk by chunk while the provider is still producing it. */
    Flux<byte[]> askStreaming(Authentication user, String conversation, Resource recording) {
        String question = this.transcription.transcribe(recording);
        String answer = answer(user, conversation, question);
        return this.speech.stream(SpokenText.forSpeech(answer, MAX_SPOKEN_CHARS));
    }

    private String answer(Authentication user, String conversation, String question) {
        if (question == null || question.isBlank()) {
            throw new IllegalArgumentException("no speech found in the recording");
        }
        return this.chatClient.prompt()
                .system("The answer will be read aloud: at most four short sentences, no lists, no code, "
                        + "no markdown. Say where a code example can be found instead of reading it.")
                .user(question)
                .advisors(spec -> spec.param(ChatMemory.CONVERSATION_ID,
                        this.conversationIds.forUser(user, conversation)))
                .call()
                .content();
    }
}

package com.example.docsassistant;

import java.util.regex.Pattern;

/** Turns a written answer into something worth reading aloud. */
final class SpokenText {

    private static final Pattern CODE_BLOCK = Pattern.compile("(?s)```.*?```");
    private static final Pattern LINK = Pattern.compile("\\[([^\\]]+)]\\([^)]*\\)");
    private static final Pattern MARKUP = Pattern.compile("[`*_#>|]");

    private SpokenText() {
    }

    static String forSpeech(String answer, int maxChars) {
        String text = CODE_BLOCK.matcher(answer).replaceAll(" (a code example is in the written answer) ");
        text = LINK.matcher(text).replaceAll("$1");
        text = MARKUP.matcher(text).replaceAll("");
        text = text.replaceAll("\\s+", " ").strip();
        if (text.length() <= maxChars) {
            return text;
        }
        int cut = text.lastIndexOf(". ", maxChars);
        return text.substring(0, cut > 0 ? cut + 1 : maxChars);
    }
}

package com.example.docsassistant;

import java.io.IOException;

import org.springframework.core.io.Resource;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.security.core.Authentication;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.server.ResponseStatusException;
import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyEmitter;
import reactor.core.Disposable;

@RestController
class VoiceController {

    private static final long MAX_BYTES = 10L * 1024 * 1024;
    private static final MediaType MPEG = MediaType.parseMediaType("audio/mpeg");

    private final VoiceAssistantService voice;

    VoiceController(VoiceAssistantService voice) {
        this.voice = voice;
    }

    /** Multipart audio in, one MP3 out. */
    @PostMapping(path = "/api/voice/ask", consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = "audio/mpeg")
    byte[] ask(@RequestParam("audio") MultipartFile audio,
            @RequestParam(name = "session", defaultValue = "voice") String session, Authentication user) {
        return this.voice.ask(user, session, checked(audio)).audio();
    }

    /** Same request, but the MP3 leaves as the provider produces it, and a listener who hangs up stops the work. */
    @PostMapping(path = "/api/voice/ask/stream", consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
            produces = "audio/mpeg")
    ResponseEntity<ResponseBodyEmitter> askStreaming(@RequestParam("audio") MultipartFile audio,
            @RequestParam(name = "session", defaultValue = "voice") String session, Authentication user) {
        ResponseBodyEmitter emitter = new ResponseBodyEmitter(120_000L);
        Disposable subscription = this.voice.askStreaming(user, session, checked(audio))
                .subscribe(
                        chunk -> send(emitter, chunk),
                        emitter::completeWithError,
                        emitter::complete);
        emitter.onCompletion(subscription::dispose);
        emitter.onError(error -> subscription.dispose());
        emitter.onTimeout(subscription::dispose);
        return ResponseEntity.ok().contentType(MPEG).body(emitter);
    }

    private static void send(ResponseBodyEmitter emitter, byte[] chunk) {
        try {
            emitter.send(chunk, MPEG);
        }
        catch (IOException | IllegalStateException ex) {
            emitter.completeWithError(ex);   // the listener left; onError cancels the provider stream
        }
    }

    private static Resource checked(MultipartFile audio) {
        if (audio.isEmpty() || audio.getSize() > MAX_BYTES) {
            throw new ResponseStatusException(HttpStatus.CONTENT_TOO_LARGE, "audio must be 1 byte to 10 MiB");
        }
        return audio.getResource();
    }
}

package com.example.docsassistant;

import static org.assertj.core.api.Assertions.assertThat;

import java.util.ArrayList;
import java.util.List;

import org.junit.jupiter.api.Test;
import org.springframework.ai.audio.transcription.AudioTranscription;
import org.springframework.ai.audio.transcription.AudioTranscriptionResponse;
import org.springframework.ai.audio.transcription.TranscriptionModel;
import org.springframework.ai.audio.tts.Speech;
import org.springframework.ai.audio.tts.TextToSpeechModel;
import org.springframework.ai.audio.tts.TextToSpeechPrompt;
import org.springframework.ai.audio.tts.TextToSpeechResponse;
import org.springframework.ai.chat.client.ChatClient;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.model.Generation;
import org.springframework.ai.chat.memory.ChatMemory;
import org.springframework.ai.chat.memory.MessageWindowChatMemory;
import org.springframework.ai.chat.client.advisor.MessageChatMemoryAdvisor;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.security.authentication.TestingAuthenticationToken;
import reactor.core.publisher.Flux;

class VoiceAssistantServiceTest {

    @Test
    void speaksTheAnswerToTheQuestionItHeard() {
        List<String> spoken = new ArrayList<>();
        TranscriptionModel stt = prompt ->
                new AudioTranscriptionResponse(new AudioTranscription("How do I add a page to the nav?"));
        TextToSpeechModel tts = new TextToSpeechModel() {
            @Override
            public TextToSpeechResponse call(TextToSpeechPrompt prompt) {
                spoken.add(prompt.getInstructions().getText());
                return new TextToSpeechResponse(List.of(new Speech(new byte[] {1, 2, 3})));
            }

            @Override
            public Flux<TextToSpeechResponse> stream(TextToSpeechPrompt prompt) {
                return Flux.just(call(prompt));
            }
        };
        ChatModel chat = prompt -> new ChatResponse(List.of(new Generation(
                new AssistantMessage("Add it to **nav.adoc**. See [the guide](https://example.org/nav)."))));
        ChatMemory memory = MessageWindowChatMemory.builder().maxMessages(10).build();
        ChatClient client = ChatClient.builder(chat)
                .defaultAdvisors(MessageChatMemoryAdvisor.builder(memory).build())
                .build();
        var service = new VoiceAssistantService(stt, tts, client, new ConversationIds());
        var user = new TestingAuthenticationToken("ana", "n/a", "ROLE_USER");

        var result = service.ask(user, "voice", new ByteArrayResource(new byte[] {1}));

        assertThat(result.question()).isEqualTo("How do I add a page to the nav?");
        assertThat(result.audio()).containsExactly(1, 2, 3);
        assertThat(spoken).containsExactly("Add it to nav.adoc. See the guide.");
    }

    @Test
    void shortensLongAnswersAtASentenceBoundary() {
        String text = "First sentence. Second sentence. " + "x".repeat(100);
        assertThat(SpokenText.forSpeech(text, 40)).isEqualTo("First sentence. Second sentence.");
    }
}