How to Build an Agentic Voice AI Assistant that Understands, Reasons, Plans, and Responds through Autonomous Multi-Step Intelligence

import subprocess
import sys
import json
import re
from datetime import datetime
from typing import Dict, List, Tuple, Any


def install_packages():
   packages = ['transformers', 'torch', 'torchaudio', 'datasets', 'soundfile',
               'librosa', 'IPython', 'numpy']
   for pkg in packages:
       subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', pkg])


print(" Initializing Agentic Voice AI...")
install_packages()


import torch
import soundfile as sf
import numpy as np
from transformers import (AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline,
                        SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan)
from IPython.show import Audio, show, HTML
import warnings
warnings.filterwarnings('ignore')

class VoiceAgent:
   def __init__(self):
       self.reminiscence = []
       self.context = {}
       self.instruments = {}
       self.targets = []
      
   def understand(self, audio_input: str) -> Dict[str, Any]:
       intent = self._extract_intent(audio_input)
       entities = self._extract_entities(audio_input)
       sentiment = self._analyze_sentiment(audio_input)
       notion = {
           'textual content': audio_input,
           'intent': intent,
           'entities': entities,
           'sentiment': sentiment,
           'timestamp': datetime.now().isoformat()
       }
       self.reminiscence.append(notion)
       return notion
  
   def _extract_intent(self, textual content: str) -> str:
       text_lower = textual content.decrease()
       intent_patterns = {
           'create': ['create', 'make', 'generate', 'write'],
           'search': ['search', 'find', 'look for', 'show me'],
           'analyze': ['analyze', 'explain', 'understand', 'what is'],
           'calculate': ['calculate', 'compute', 'how much', 'sum'],
           'schedule': ['schedule', 'plan', 'set reminder', 'meeting'],
           'translate': ['translate', 'say in', 'convert to'],
           'summarize': ['summarize', 'brief', 'tldr', 'overview']
       }
       for intent, key phrases in intent_patterns.gadgets():
           if any(kw in text_lower for kw in key phrases):
               return intent
       return 'dialog'
  
   def _extract_entities(self, textual content: str) -> Dict[str, List[str]]:
       entities = {
           'numbers': re.findall(r'd+', textual content),
           'dates': re.findall(r'bd{1,2}/d{1,2}/d{2,4}b', textual content),
           'instances': re.findall(r'bd{1,2}:d{2}s*(?:am|pm)?b', textual content.decrease()),
           'emails': re.findall(r'b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+.[A-Z|a-z]{2,}b', textual content)
       }
       return {ok: v for ok, v in entities.gadgets() if v}
  
   def _analyze_sentiment(self, textual content: str) -> str:
       constructive = ['good', 'great', 'excellent', 'happy', 'love', 'thank']
       detrimental = ['bad', 'terrible', 'sad', 'hate', 'angry', 'problem']
       text_lower = textual content.decrease()
       pos_count = sum(1 for phrase in constructive if phrase in text_lower)
       neg_count = sum(1 for phrase in detrimental if phrase in text_lower)
       if pos_count > neg_count:
           return 'constructive'
       elif neg_count > pos_count:
           return 'detrimental'
       return 'impartial'

def motive(self, notion: Dict) -> Dict[str, Any]:
       intent = notion['intent']
       reasoning = {
           'objective': self._identify_goal(intent),
           'conditions': self._check_prerequisites(intent),
           'plan': self._create_plan(intent, notion['entities']),
           'confidence': self._calculate_confidence(notion)
       }
       return reasoning
  
   def act(self, reasoning: Dict) -> str:
       plan = reasoning['plan']
       outcomes = []
       for step in plan['steps']:
           outcome = self._execute_step(step)
           outcomes.append(outcome)
       response = self._generate_response(outcomes, reasoning)
       return response
  
   def _identify_goal(self, intent: str) -> str:
       goal_mapping = {
           'create': 'Generate new content material',
           'search': 'Retrieve data',
           'analyze': 'Understand and clarify',
           'calculate': 'Perform computation',
           'schedule': 'Organize time-based duties',
           'translate': 'Convert between languages',
           'summarize': 'Condense data'
       }
       return goal_mapping.get(intent, 'Assist consumer')
  
   def _check_prerequisites(self, intent: str) -> List[str]:
       prereqs = {
           'search': ['internet access', 'search tool'],
           'calculate': ['math processor'],
           'translate': ['translation model'],
           'schedule': ['calendar access']
       }
       return prereqs.get(intent, ['language understanding'])
  
   def _create_plan(self, intent: str, entities: Dict) -> Dict:
       plans = {
           'create': {'steps': ['understand_requirements', 'generate_content', 'validate_output'], 'estimated_time': '10s'},
           'analyze': {'steps': ['parse_input', 'analyze_components', 'synthesize_explanation'], 'estimated_time': '5s'},
           'calculate': {'steps': ['extract_numbers', 'determine_operation', 'compute_result'], 'estimated_time': '2s'}
       }
       default_plan = {'steps': ['understand_query', 'process_information', 'formulate_response'], 'estimated_time': '3s'}
       return plans.get(intent, default_plan)

 def _calculate_confidence(self, notion: Dict) -> float:
       base_confidence = 0.7
       if notion['entities']:
           base_confidence += 0.15
       if notion['sentiment'] != 'impartial':
           base_confidence += 0.1
       if len(notion['text'].cut up()) > 5:
           base_confidence += 0.05
       return min(base_confidence, 1.0)
  
   def _execute_step(self, step: str) -> Dict:
       return {'step': step, 'standing': 'accomplished', 'output': f'Executed {step}'}
  
   def _generate_response(self, outcomes: List, reasoning: Dict) -> str:
       intent = reasoning['goal']
       confidence = reasoning['confidence']
       prefix = "I perceive you need to" if confidence > 0.8 else "I feel you are asking me to"
       response = f"{prefix} {intent.decrease()}. "
       if len(self.reminiscence) > 1:
           response += "Based on our dialog, "
       response += f"I've analyzed your request and accomplished {len(outcomes)} steps. "
       return response

class VoiceIO:
   def __init__(self):
       print("Loading voice fashions...")
       system = "cuda:0" if torch.cuda.is_available() else "cpu"
       self.stt_pipe = pipeline("automatic-speech-recognition", mannequin="openai/whisper-base", system=system)
       self.tts_processor = SpeechT5Processor.from_pretrained("microsoft/speecht5_tts")
       self.tts_model = SpeechT5ForTextToSpeech.from_pretrained("microsoft/speecht5_tts")
       self.vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")
       self.speaker_embeddings = torch.randn(1, 512) * 0.1
       print("✓ Voice I/O prepared")
  
   def pay attention(self, audio_path: str) -> str:
       outcome = self.stt_pipe(audio_path)
       return outcome['text']
  
   def communicate(self, textual content: str, output_path: str = "response.wav") -> Tuple[str, np.ndarray]:
       inputs = self.tts_processor(textual content=textual content, return_tensors="pt")
       speech = self.tts_model.generate_speech(inputs["input_ids"], self.speaker_embeddings, vocoder=self.vocoder)
       sf.write(output_path, speech.numpy(), samplerate=16000)
       return output_path, speech.numpy()




class AgenticVoiceAssistant:
   def __init__(self):
       self.agent = VoiceAgent()
       self.voice_io = VoiceIO()
       self.interaction_count = 0
      
   def process_voice_input(self, audio_path: str) -> Dict:
       text_input = self.voice_io.pay attention(audio_path)
       notion = self.agent.understand(text_input)
       reasoning = self.agent.motive(notion)
       response_text = self.agent.act(reasoning)
       audio_path, audio_array = self.voice_io.communicate(response_text)
       self.interaction_count += 1
       return {
           'input_text': text_input,
           'notion': notion,
           'reasoning': reasoning,
           'response_text': response_text,
           'audio_path': audio_path,
           'audio_array': audio_array
       }

  def display_reasoning(self, outcome: Dict):
       html = f"""
       <div model='background: #1e1e1e; colour: #fff; padding: 20px; border-radius: 10px; font-family: monospace;'>
           <h2 model='colour: #4CAF50;'> Agent Reasoning Process</h2>
           <div><robust model='colour: #2196F3;'> INPUT:</robust> {outcome['input_text']}</div>
           <div><robust model='colour: #FF9800;'> PERCEPTION:</robust>
               <ul>
                   <li>Intent: {outcome['perception']['intent']}</li>
                   <li>Entities: {outcome['perception']['entities']}</li>
                   <li>Sentiment: {outcome['perception']['sentiment']}</li>
               </ul>
           </div>
           <div><robust model='colour: #9C27B0;'> REASONING:</robust>
               <ul>
                   <li>Goal: {outcome['reasoning']['goal']}</li>
                   <li>Plan: {len(outcome['reasoning']['plan']['steps'])} steps</li>
                   <li>Confidence: {outcome['reasoning']['confidence']:.2%}</li>
               </ul>
           </div>
           <div><robust model='colour: #4CAF50;'> RESPONSE:</robust> {outcome['response_text']}</div>
       </div>
       """
       show(HTML(html))




def run_agentic_demo():
   print("n" + "="*70)
   print(" AGENTIC VOICE AI ASSISTANT")
   print("="*70 + "n")
   assistant = AgenticVoiceAssistant()
   situations = [
       "Create a summary of machine learning concepts",
       "Calculate the sum of twenty five and thirty seven",
       "Analyze the benefits of renewable energy"
   ]
   for i, scenario_text in enumerate(situations, 1):
       print(f"n--- Scenario {i} ---")
       print(f"Simulated Input: '{scenario_text}'")
       audio_path, _ = assistant.voice_io.communicate(scenario_text, f"input_{i}.wav")
       outcome = assistant.process_voice_input(audio_path)
       assistant.display_reasoning(outcome)
       print("n Playing agent's voice response...")
       show(Audio(outcome['audio_array'], price=16000))
       print("n" + "-"*70)
   print(f"n Completed {assistant.interaction_count} agentic interactions")
   print("n Key Agentic Capabilities Demonstrated:")
   print("  • Autonomous notion and understanding")
   print("  • Intent recognition and entity extraction")
   print("  • Multi-step reasoning and planning")
   print("  • Goal-driven motion execution")
   print("  • Natural language response technology")
   print("  • Memory and context administration")


if __name__ == "__main__":
   run_agentic_demo()