Merge pull request #390 from F4310/play_sound

Playing Audio (WAV) through Vector

chipper/pkg/wirepod/sdkapp/server.go

@@ -140,6 +140,59 @@ func SdkapiHandler(w http.ResponseWriter, r *http.Request) {
 			w.Write([]byte(json))
 			return
 		}
+
+	case r.URL.Path == "/api-sdk/play_sound":
+		file, _, err := r.FormFile("sound")
+		if err != nil {
+			println("Error retrieving the file:", err)
+			return
+		}
+		defer file.Close()
+
+		// Lê o conteúdo do arquivo em um slice de bytes
+		pcmFile, err := io.ReadAll(file)
+		if err != nil {
+			println("Error reading the file:", err)
+			return
+		}
+
+		var audioChunks [][]byte
+		for len(pcmFile) >= 1024 {
+			audioChunks = append(audioChunks, pcmFile[:1024])
+			pcmFile = pcmFile[1024:]
+		}
+
+		var audioClient vectorpb.ExternalInterface_ExternalAudioStreamPlaybackClient
+		audioClient, _ = robot.Conn.ExternalAudioStreamPlayback(ctx)
+		audioClient.SendMsg(&vectorpb.ExternalAudioStreamRequest{
+			AudioRequestType: &vectorpb.ExternalAudioStreamRequest_AudioStreamPrepare{
+				AudioStreamPrepare: &vectorpb.ExternalAudioStreamPrepare{
+					AudioFrameRate: 8000,
+					AudioVolume:    uint32(100),
+				},
+			},
+		})
+
+		for _, chunk := range audioChunks {
+			audioClient.SendMsg(&vectorpb.ExternalAudioStreamRequest{
+				AudioRequestType: &vectorpb.ExternalAudioStreamRequest_AudioStreamChunk{
+					AudioStreamChunk: &vectorpb.ExternalAudioStreamChunk{
+						AudioChunkSizeBytes: uint32(len(chunk)),
+						AudioChunkSamples:   chunk,
+					},
+				},
+			})
+			time.Sleep(time.Millisecond * 60)
+		}
+
+		audioClient.SendMsg(&vectorpb.ExternalAudioStreamRequest{
+			AudioRequestType: &vectorpb.ExternalAudioStreamRequest_AudioStreamComplete{
+				AudioStreamComplete: &vectorpb.ExternalAudioStreamComplete{},
+			},
+		})
+
+		return
+
 	case r.URL.Path == "/api-sdk/get_battery":
 		// Ensure the endpoint times out after 15 seconds
 		ctx := r.Context() // Get the request context
@@ -573,3 +626,58 @@ func BeginServer() {
 		}
 	}
 }
+
+func rgbToBytes(rgbValues [][][3]uint8) ([]byte, error) {
+	var buffer bytes.Buffer
+
+	for _, row := range rgbValues {
+		for _, pixel := range row {
+			// Directly add the R, G and B values ​​to the buffer
+			buffer.WriteByte(pixel[0]) // R
+			buffer.WriteByte(pixel[1]) // G
+			buffer.WriteByte(pixel[2]) // B
+		}
+	}
+
+	return buffer.Bytes(), nil
+}
+func imageToBytes(img image.Image) ([]byte, error) {
+	bounds := img.Bounds()
+	var buffer bytes.Buffer
+
+	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
+		for x := bounds.Min.X; x < bounds.Max.X; x++ {
+			// Obtém a cor do pixel
+			c := img.At(x, y)
+			r, g, b, _ := c.RGBA() // Ignorando o valor Alpha
+
+			// Converte de uint32 para uint8
+			buffer.WriteByte(uint8(r >> 8))
+			buffer.WriteByte(uint8(g >> 8))
+			buffer.WriteByte(uint8(b >> 8))
+		}
+	}
+
+	return buffer.Bytes(), nil
+}
+
+func resizeImage(original image.Image, width, height int) image.Image {
+	if width <= 0 || height <= 0 {
+		return original
+	}
+
+	newImage := image.NewRGBA(image.Rect(0, 0, width, height))
+
+	scaleX := float64(original.Bounds().Dx()) / float64(width)
+	scaleY := float64(original.Bounds().Dy()) / float64(height)
+
+	for y := 0; y < height; y++ {
+		for x := 0; x < width; x++ {
+			srcX := int(float64(x) * scaleX)
+			srcY := int(float64(y) * scaleY)
+			newImage.Set(x, y, original.At(srcX, srcY))
+		}
+	}
+
+	return newImage
+}

chipper/webroot/js/play_audio.js

@@ -0,0 +1,126 @@
+
+//Convert the WAV frame rate on the client machine and send the WAV file to be processed at /api-sdk/play_sound
+
+let processedAudioBlob = null; 
+
+document.getElementById('fileInput').addEventListener('change', async () => {
+    const fileInput = document.getElementById('fileInput');
+    if (!fileInput.files.length) {
+        alert('Please, select a WAV file');
+        return;
+    }
+
+    const file = fileInput.files[0];
+    const arrayBuffer = await file.arrayBuffer();
+    const audioContext = new (window.AudioContext || window.webkitAudioContext)();
+
+    try {
+        const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
+
+        // adjust frame rate to 8000 Hz
+        const newSampleRate = 8000;
+        const newLength = Math.round(audioBuffer.length * newSampleRate / audioBuffer.sampleRate);
+        const newBuffer = audioContext.createBuffer(audioBuffer.numberOfChannels, newLength, newSampleRate);
+
+        for (let channel = 0; channel < audioBuffer.numberOfChannels; channel++) {
+            const oldData = audioBuffer.getChannelData(channel);
+            const newData = newBuffer.getChannelData(channel);
+
+            for (let i = 0; i < newLength; i++) {
+                const oldIndex = i * audioBuffer.sampleRate / newSampleRate;
+                const index0 = Math.floor(oldIndex);
+                const index1 = Math.min(index0 + 1, oldData.length - 1);
+                const fraction = oldIndex - index0;
+
+                newData[i] = oldData[index0] * (1 - fraction) + oldData[index1] * fraction; 
+            }
+        }
+
+        // Create a new WAV file
+        processedAudioBlob = await bufferToWave(newBuffer);
+        const url = URL.createObjectURL(processedAudioBlob);
+
+        // play processed audio
+        const audioOutput = document.getElementById('audioOutput');
+        audioOutput.src = url;
+        audioOutput.play();
+
+        // show send button
+        document.getElementById('uploadButton').style.display = 'inline-block';
+    } catch (error) {
+        console.error('Error processing the file:', error);
+    }
+});
+
+function bufferToWave(abuffer) {
+    const numOfChannels = abuffer.numberOfChannels;
+    const length = abuffer.length * numOfChannels * 2 + 44;
+    const buffer = new ArrayBuffer(length);
+    const view = new DataView(buffer);
+    let offset = 0;
+
+    // Escrever cabeçalho WAV
+    setString(view, offset, 'RIFF'); offset += 4;
+    view.setUint32(offset, length - 8, true); offset += 4;
+    setString(view, offset, 'WAVE'); offset += 4;
+    setString(view, offset, 'fmt '); offset += 4;
+    view.setUint32(offset, 16, true); offset += 4; // format size
+    view.setUint16(offset, 1, true); offset += 2; // PCM format
+    view.setUint16(offset, numOfChannels, true); offset += 2; // number of channels
+    view.setUint32(offset, 8000, true); offset += 4; // sample rate
+    view.setUint32(offset, 8000 * numOfChannels * 2, true); offset += 4; // byte rate
+    view.setUint16(offset, numOfChannels * 2, true); offset += 2; // block align
+    view.setUint16(offset, 16, true); offset += 2; // bits per sample
+
+    setString(view, offset, 'data'); offset += 4;
+    view.setUint32(offset, length - offset - 4, true); offset += 4;
+
+    // Copiar os dados de áudio
+    for (let channel = 0; channel < numOfChannels; channel++) {
+        const channelData = abuffer.getChannelData(channel);
+        for (let i = 0; i < channelData.length; i++) {
+            view.setInt16(offset, channelData[i] * 0x7FFF, true);
+            offset += 2;
+        }
+    }
+
+    return new Blob([buffer], { type: 'audio/wav' });
+}
+
+function setString(view, offset, string) {
+    for (let i = 0; i < string.length; i++) {
+        view.setUint8(offset + i, string.charCodeAt(i));
+    }
+}
+
+document.getElementById('uploadButton').addEventListener('click', async () => {
+    if (!processedAudioBlob) {
+        alert('No processed audio to send.');
+        return;
+    }
+
+    await uploadAudio(processedAudioBlob);
+});
+
+async function uploadAudio(blob) {
+    const formData = new FormData();
+    formData.append('sound', blob, 'processed.wav');
+
+    try {
+        const dominio = window.location.hostname;
+        esn = urlParams.get("serial");
+        const response = await fetch('/api-sdk/play_sound?serial='+esn, {
+            method: 'POST',
+            body: formData,
+        });
+
+        if (!response.ok) {
+            throw new Error('Erro to send audio: ' + response.statusText);
+        }
+
+        const result = await response.json();
+        console.log('Audio sent successfully:', result);
+    } catch (error) {
+        console.error('Error sending audio:', error);
+    }
+}

chipper/webroot/sdkapp/control.html

@@ -86,9 +86,19 @@ <h2 class="center">Say Text</h2>
           <input class="tinput" name="textSay" id="textSay" type="text" placeholder="Put text here" disabled />
         </div>
         <button onclick="sayText()" type="submit">Submit</button>
+      <hr />
+      <h2 class="center">Play Audio</h2>
+        <div class="center">
+          <input type="file" id="fileInput" accept=".wav">
+        </div>
+        <button id="uploadButton" style="display: none;" type="submit">Send Audio</button>
+        <div div class="center">
+          <audio id="audioOutput" controls></audio>
+        </div>
       </div>
       <hr />
     </div>
+    </div>
   </div>
   <hr>
   </div>
@@ -98,6 +108,8 @@ <h2 class="center">Say Text</h2>
   <script src="./js/iro.min.js"></script>
   <script src="./js/control.js"></script>
   <script src="../js/ui.js"></script>
+  <script src="../js/play_audio.js"></script>
+
   <iframe name="hiddenFrame" width="0" height="0" border="0" style="display: none"></iframe>
 </body>