Feat : text to speech & flight api

2025-03-02 20:51:50 +01:00
parent c2c1c7f09f
commit 0c0536c43a
9 changed files with 306 additions and 8 deletions
@@ -2,6 +2,8 @@
 from sources.utility import pretty_print
 from sources.agent import Agent
 from sources.tools.webSearch import webSearch
+from sources.tools.flightSearch import FlightSearch
+
 class CasualAgent(Agent):
    def __init__(self, model, name, prompt_path, provider):
        """
@@ -9,7 +11,8 @@ class CasualAgent(Agent):
        """
        super().__init__(model, name, prompt_path, provider)
        self.tools = {
-            "web_search": webSearch()
+            "web_search": webSearch(),
+            "flight_search": FlightSearch()
        }
        self.role = "talking"
    
@@ -2,9 +2,13 @@
 from sources.text_to_speech import Speech
 from sources.utility import pretty_print
 from sources.router import AgentRouter
+from sources.speech_to_text import AudioTranscriber, AudioRecorder

 class Interaction:
-    def __init__(self, agents, tts_enabled: bool = False, recover_last_session: bool = False):
+    def __init__(self, agents,
+                 tts_enabled: bool = True,
+                 stt_enabled: bool = True,
+                 recover_last_session: bool = False):
        self.tts_enabled = tts_enabled
        self.agents = agents
        self.current_agent = None
@@ -13,11 +17,25 @@ class Interaction:
        self.is_active = True
        self.last_query = None
        self.last_answer = None
+        self.ai_name = self.find_ai_name()
+        self.tts_enabled = tts_enabled
+        self.stt_enabled = stt_enabled
+        if stt_enabled:
+            self.transcriber = AudioTranscriber(self.ai_name, verbose=False)
+            self.recorder = AudioRecorder()
        if tts_enabled:
            self.speech.speak("Hello Sir, we are online and ready. What can I do for you ?")
        if recover_last_session:
            self.recover_last_session()
    
+    def find_ai_name(self) -> str:
+        ai_name = "jarvis"
+        for agent in self.agents:
+            if agent.role == "talking":
+                ai_name = agent.agent_name
+                break
+        return ai_name
+    
    def recover_last_session(self):
        for agent in self.agents:
            agent.memory.load_memory()
@@ -36,9 +54,22 @@ class Interaction:
            if buffer == "exit" or buffer == "goodbye":
                return None
        return buffer
+    
+    def transcription_job(self):
+        self.recorder = AudioRecorder()
+        self.transcriber = AudioTranscriber(self.ai_name, verbose=False)
+        self.transcriber.start()
+        self.recorder.start()
+        self.recorder.join()
+        self.transcriber.join()
+        query = self.transcriber.get_transcript()
+        return query

    def get_user(self):
-        query = self.read_stdin()
+        if self.stt_enabled:
+            query = self.transcription_job()
+        else:
+            query = self.read_stdin()
        if query is None:
            self.is_active = False
            self.last_query = "Goodbye (exit requested by user, dont think, make answer very short)"
@@ -47,6 +78,8 @@ class Interaction:
        return query
    
    def think(self):
+        if self.last_query is None:
+            return
        agent = self.router.select_agent(self.last_query)
        if self.current_agent != agent:
            self.current_agent = agent
@@ -55,6 +88,8 @@ class Interaction:
        self.last_answer, _ = agent.process(self.last_query, self.speech)
    
    def show_answer(self):
+        if self.last_query is None:
+            return
        self.current_agent.show_answer()
        if self.tts_enabled:
            self.speech.speak(self.last_answer)
@@ -26,6 +26,8 @@ class AgentRouter:
        return result
    
    def select_agent(self, text: str) -> Agent:
+        if text is None:
+            return self.agents[0]
        result = self.classify_text(text)
        for agent in self.agents:
            if result["labels"][0] == agent.role:
@@ -0,0 +1,165 @@
+from colorama import Fore
+import pyaudio
+import queue
+import threading
+import numpy as np
+import torch
+from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline
+import time
+import librosa
+
+audio_queue = queue.Queue()
+done = False
+
+class AudioRecorder:
+    def __init__(self, format=pyaudio.paInt16, channels=1, rate=44100, chunk=8192, record_seconds=7, verbose=False):
+        self.format = format
+        self.channels = channels
+        self.rate = rate
+        self.chunk = chunk
+        self.record_seconds = record_seconds
+        self.verbose = verbose
+        self.audio = pyaudio.PyAudio()
+        self.thread = threading.Thread(target=self._record, daemon=True)
+
+    def _record(self):
+        stream = self.audio.open(format=self.format, channels=self.channels, rate=self.rate,
+                                 input=True, frames_per_buffer=self.chunk)
+        if self.verbose:
+            print(Fore.GREEN + "AudioRecorder: Started recording..." + Fore.RESET)
+
+        while not done:
+            frames = []
+            for _ in range(0, int(self.rate / self.chunk * self.record_seconds)):
+                try:
+                    data = stream.read(self.chunk, exception_on_overflow=False)
+                    frames.append(data)
+                except Exception as e:
+                    print(Fore.RED + f"AudioRecorder: Failed to read stream - {e}" + Fore.RESET)
+            
+            raw_data = b''.join(frames)
+            audio_data = np.frombuffer(raw_data, dtype=np.int16)
+            audio_queue.put((audio_data, self.rate))
+            if self.verbose:
+                print(Fore.GREEN + "AudioRecorder: Added audio chunk to queue" + Fore.RESET)
+
+        stream.stop_stream()
+        stream.close()
+        self.audio.terminate()
+        if self.verbose:
+            print(Fore.GREEN + "AudioRecorder: Stopped" + Fore.RESET)
+
+    def start(self):
+        """Start the recording thread."""
+        self.thread.start()
+
+    def join(self):
+        """Wait for the recording thread to finish."""
+        self.thread.join()
+
+class Transcript:
+    def __init__(self) -> None:
+        self.last_read = None
+        device = "cuda:0" if torch.cuda.is_available() else "cpu"
+        torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
+        model_id = "distil-whisper/distil-medium.en"
+        
+        model = AutoModelForSpeechSeq2Seq.from_pretrained(
+            model_id, torch_dtype=torch_dtype, use_safetensors=True
+        )
+        model.to(device)
+        processor = AutoProcessor.from_pretrained(model_id)
+        
+        self.pipe = pipeline(
+            "automatic-speech-recognition",
+            model=model,
+            tokenizer=processor.tokenizer,
+            feature_extractor=processor.feature_extractor,
+            max_new_tokens=128,
+            torch_dtype=torch_dtype,
+            device=device,
+        )
+
+    def transcript_job(self, audio_data: np.ndarray, sample_rate: int = 16000):
+        if audio_data.dtype != np.float32:
+            audio_data = audio_data.astype(np.float32) / np.iinfo(audio_data.dtype).max
+        if len(audio_data.shape) > 1:
+            audio_data = np.mean(audio_data, axis=1)
+        if sample_rate != 16000:
+            audio_data = librosa.resample(audio_data, orig_sr=sample_rate, target_sr=16000)
+        result = self.pipe(audio_data)
+        return result["text"]
+
+class AudioTranscriber:
+    def __init__(self, ai_name: str, verbose=False):
+        self.verbose = verbose
+        self.ai_name = ai_name
+        self.transcriptor = Transcript()
+        self.thread = threading.Thread(target=self._transcribe, daemon=True)
+        self.trigger_words = {
+            'EN': [f"{self.ai_name}"],
+            'FR': [f"{self.ai_name}"],
+            'ZH': [f"{self.ai_name}"],
+            'ES': [f"{self.ai_name}"]
+        }
+        self.confirmation_words = {
+            'EN': ["do it", "go ahead", "execute", "run", "start", "thanks", "would ya", "please", "okay?", "proceed", "continue", "go on", "do that", "do that thing"],
+            'FR': ["fais-le", "vas-y", "exécute", "lance", "commence", "merci", "tu veux bien", "s'il te plaît", "d'accord ?", "poursuis", "continue", "vas-y", "fais ça", "fais ce truc"],
+            'ZH': ["做吧", "继续", "执行", "运行", "开始", "谢谢", "可以吗", "请", "好吗", "进行", "继续", "往前走", "做那个", "做那件事"],
+            'ES': ["hazlo", "adelante", "ejecuta", "corre", "empieza", "gracias", "lo harías", "por favor", "¿vale?", "procede", "continúa", "sigue", "haz eso", "haz esa cosa"]
+        }
+        self.recorded = ""
+
+    def get_transcript(self):
+        buffer = self.recorded
+        self.recorded = ""
+        return buffer
+
+    def _transcribe(self):
+        global done
+        if self.verbose:
+            print(Fore.BLUE + "AudioTranscriber: Started processing..." + Fore.RESET)
+        
+        while not done or not audio_queue.empty():
+            try:
+                audio_data, sample_rate = audio_queue.get(timeout=1.0)
+                if self.verbose:
+                    print(Fore.BLUE + "AudioTranscriber: Processing audio chunk" + Fore.RESET)
+                
+                text = self.transcriptor.transcript_job(audio_data, sample_rate)
+                self.recorded += text
+                print(Fore.YELLOW + f"Transcribed: {text}" + Fore.RESET)
+                for language, words in self.trigger_words.items():
+                    if any(word in text.lower() for word in words):
+                        print(Fore.GREEN + f"Start listening..." + Fore.RESET)
+                        self.recorded = text
+                for language, words in self.confirmation_words.items():
+                    if any(word in text.lower() for word in words):
+                        print(Fore.GREEN + f"Trigger detected. Sending to AI..." + Fore.RESET)
+                        audio_queue.task_done()
+                        done = True
+                        break
+            except queue.Empty:
+                time.sleep(0.1)
+                continue
+            except Exception as e:
+                print(Fore.RED + f"AudioTranscriber: Error - {e}" + Fore.RESET)
+        if self.verbose:
+            print(Fore.BLUE + "AudioTranscriber: Stopped" + Fore.RESET)
+
+    def start(self):
+        """Start the transcription thread."""
+        self.thread.start()
+
+    def join(self):
+        """Wait for the transcription thread to finish."""
+        self.thread.join()
+
+
+if __name__ == "__main__":
+    recorder = AudioRecorder(verbose=True)
+    transcriber = AudioTranscriber(verbose=True, ai_name="jarvis")
+    recorder.start()
+    transcriber.start()
+    recorder.join()
+    transcriber.join()
@@ -26,6 +26,8 @@ class BashInterpreter(Tools):
    
        concat_output = ""
        for command in commands:
+            if "python3" in command:
+                continue # because stubborn AI always want to run python3 with bash when it write code
            try:
                process = subprocess.Popen(
                    command,
@@ -0,0 +1,83 @@
+import os
+import requests
+import dotenv
+
+dotenv.load_dotenv()
+
+if __name__ == "__main__":
+    from tools import Tools
+else:
+    from sources.tools.tools import Tools
+
+class FlightSearch(Tools):
+    def __init__(self, api_key: str = None):
+        """
+        A tool to search for flight information using a flight number via AviationStack API.
+        """
+        super().__init__()
+        self.tag = "flight_search"
+        self.api_key = api_key or os.getenv("AVIATIONSTACK_API_KEY")
+
+    def execute(self, blocks: str, safety: bool = True) -> str:
+        if self.api_key is None:
+            return "Error: No AviationStack API key provided."
+        
+        for block in blocks:
+            flight_number = block.strip()
+            if not flight_number:
+                return "Error: No flight number provided."
+
+            try:
+                url = "http://api.aviationstack.com/v1/flights"
+                params = {
+                    "access_key": self.api_key,
+                    "flight_iata": flight_number,
+                    "limit": 1
+                }
+                response = requests.get(url, params=params)
+                response.raise_for_status()
+
+                data = response.json()
+                if "data" in data and len(data["data"]) > 0:
+                    flight = data["data"][0]
+                    # Extract key flight information
+                    flight_status = flight.get("flight_status", "Unknown")
+                    departure = flight.get("departure", {})
+                    arrival = flight.get("arrival", {})
+                    airline = flight.get("airline", {}).get("name", "Unknown")
+                    
+                    departure_airport = departure.get("airport", "Unknown")
+                    departure_time = departure.get("scheduled", "Unknown")
+                    arrival_airport = arrival.get("airport", "Unknown")
+                    arrival_time = arrival.get("scheduled", "Unknown")
+
+                    return (
+                        f"Flight: {flight_number}\n"
+                        f"Airline: {airline}\n"
+                        f"Status: {flight_status}\n"
+                        f"Departure: {departure_airport} at {departure_time}\n"
+                        f"Arrival: {arrival_airport} at {arrival_time}"
+                    )
+                else:
+                    return f"No flight information found for {flight_number}"
+            except requests.RequestException as e:
+                return f"Error during flight search: {str(e)}"
+            except Exception as e:
+                return f"Unexpected error: {str(e)}"
+        return "No flight search performed"
+
+    def execution_failure_check(self, output: str) -> bool:
+        return output.startswith("Error") or "No flight information found" in output
+
+    def interpreter_feedback(self, output: str) -> str:
+        if self.execution_failure_check(output):
+            return f"Flight search failed: {output}"
+        return f"Flight information:\n{output}"
+
+
+if __name__ == "__main__":
+    flight_tool = FlightSearch()
+    flight_number = "AA123"
+    result = flight_tool.execute([flight_number], safety=True)
+    feedback = flight_tool.interpreter_feedback(result)
+    print(feedback)
@@ -1,6 +1,9 @@

 import os
 import requests
+import dotenv
+
+dotenv.load_dotenv()

 if __name__ == "__main__":
    from tools import Tools
@@ -15,10 +18,10 @@ class webSearch(Tools):
        super().__init__()
        self.tag = "web_search"
        self.api_key = api_key or os.getenv("SERPAPI_KEY")  # Requires a SerpApi key
-        if not self.api_key:
-            raise ValueError("SerpApi key is required for webSearch tool. Set SERPAPI_KEY environment variable or pass it to the constructor.")

    def execute(self, blocks: str, safety: bool = True) -> str:
+        if self.api_key is None:
+            return "Error: No SerpApi key provided."
        for block in blocks:
            query = block.strip()
            if not query:
@@ -60,7 +63,7 @@ class webSearch(Tools):


 if __name__ == "__main__":
-    search_tool = webSearch(api_key="c4da252b63b0fc3cbf2c7dd98b931ae632aecf3feacbbfe099e17872eb192c44")
+    search_tool = webSearch(api_key=os.getenv("SERPAPI_KEY"))
    query = "when did covid start"
    result = search_tool.execute(query, safety=True)
    feedback = search_tool.interpreter_feedback(result)