1255 lines
46 KiB
JavaScript
1255 lines
46 KiB
JavaScript
|
const SAMPLE_RATE = 16000;
|
|||
|
|
const CHANNELS = 1;
|
|||
|
|
const FRAME_SIZE = 960; // 对应于60ms帧大小 (16000Hz * 0.06s = 960 samples)
|
|||
|
|
const OPUS_APPLICATION = 2049; // OPUS_APPLICATION_AUDIO
|
|||
|
|
const BUFFER_SIZE = 4096;
|
|||
|
|
|
|||
|
|
// WebSocket相关变量
|
|||
|
|
let websocket = null;
|
|||
|
|
let isConnected = false;
|
|||
|
|
|
|||
|
|
let audioContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: SAMPLE_RATE });
|
|||
|
|
let mediaStream, mediaSource, audioProcessor;
|
|||
|
|
let recordedPcmData = []; // 存储原始PCM数据
|
|||
|
|
let recordedOpusData = []; // 存储Opus编码后的数据
|
|||
|
|
let opusEncoder, opusDecoder;
|
|||
|
|
let isRecording = false;
|
|||
|
|
|
|||
|
|
const startButton = document.getElementById("start");
|
|||
|
|
const stopButton = document.getElementById("stop");
|
|||
|
|
const playButton = document.getElementById("play");
|
|||
|
|
const statusLabel = document.getElementById("status");
|
|||
|
|
|
|||
|
|
// 添加WebSocket界面元素引用
|
|||
|
|
const connectButton = document.getElementById("connectButton") || document.createElement("button");
|
|||
|
|
const serverUrlInput = document.getElementById("serverUrl") || document.createElement("input");
|
|||
|
|
const connectionStatus = document.getElementById("connectionStatus") || document.createElement("span");
|
|||
|
|
const sendTextButton = document.getElementById("sendTextButton") || document.createElement("button");
|
|||
|
|
const messageInput = document.getElementById("messageInput") || document.createElement("input");
|
|||
|
|
const conversationDiv = document.getElementById("conversation") || document.createElement("div");
|
|||
|
|
|
|||
|
|
// 添加连接和发送事件监听
|
|||
|
|
if(connectButton.id === "connectButton") {
|
|||
|
|
connectButton.addEventListener("click", connectToServer);
|
|||
|
|
}
|
|||
|
|
if(sendTextButton.id === "sendTextButton") {
|
|||
|
|
sendTextButton.addEventListener("click", sendTextMessage);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
startButton.addEventListener("click", startRecording);
|
|||
|
|
stopButton.addEventListener("click", stopRecording);
|
|||
|
|
playButton.addEventListener("click", playRecording);
|
|||
|
|
|
|||
|
|
// 音频缓冲和播放管理
|
|||
|
|
let audioBufferQueue = []; // 存储接收到的音频包
|
|||
|
|
let isAudioBuffering = false; // 是否正在缓冲音频
|
|||
|
|
let isAudioPlaying = false; // 是否正在播放音频
|
|||
|
|
const BUFFER_THRESHOLD = 3; // 缓冲包数量阈值，至少累积5个包再开始播放
|
|||
|
|
const MIN_AUDIO_DURATION = 0.1; // 最小音频长度(秒)，小于这个长度的音频会被合并
|
|||
|
|
let streamingContext = null; // 音频流上下文
|
|||
|
|
|
|||
|
|
// 初始化Opus编码器与解码器
|
|||
|
|
async function initOpus() {
|
|||
|
|
if (typeof window.ModuleInstance === 'undefined') {
|
|||
|
|
if (typeof Module !== 'undefined') {
|
|||
|
|
// 尝试使用全局Module
|
|||
|
|
window.ModuleInstance = Module;
|
|||
|
|
console.log('使用全局Module作为ModuleInstance');
|
|||
|
|
} else {
|
|||
|
|
console.error("Opus库未加载，ModuleInstance和Module对象都不存在");
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
const mod = window.ModuleInstance;
|
|||
|
|
|
|||
|
|
// 创建编码器
|
|||
|
|
opusEncoder = {
|
|||
|
|
channels: CHANNELS,
|
|||
|
|
sampleRate: SAMPLE_RATE,
|
|||
|
|
frameSize: FRAME_SIZE,
|
|||
|
|
maxPacketSize: 4000,
|
|||
|
|
module: mod,
|
|||
|
|
|
|||
|
|
// 初始化编码器
|
|||
|
|
init: function() {
|
|||
|
|
// 获取编码器大小
|
|||
|
|
const encoderSize = mod._opus_encoder_get_size(this.channels);
|
|||
|
|
console.log(`Opus编码器大小: ${encoderSize}字节`);
|
|||
|
|
|
|||
|
|
// 分配内存
|
|||
|
|
this.encoderPtr = mod._malloc(encoderSize);
|
|||
|
|
if (!this.encoderPtr) {
|
|||
|
|
throw new Error("无法分配编码器内存");
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 初始化编码器
|
|||
|
|
const err = mod._opus_encoder_init(
|
|||
|
|
this.encoderPtr,
|
|||
|
|
this.sampleRate,
|
|||
|
|
this.channels,
|
|||
|
|
OPUS_APPLICATION
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
if (err < 0) {
|
|||
|
|
throw new Error(`Opus编码器初始化失败: ${err}`);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return true;
|
|||
|
|
},
|
|||
|
|
|
|||
|
|
// 编码方法
|
|||
|
|
encode: function(pcmData) {
|
|||
|
|
const mod = this.module;
|
|||
|
|
|
|||
|
|
// 为PCM数据分配内存
|
|||
|
|
const pcmPtr = mod._malloc(pcmData.length * 2); // Int16 = 2字节
|
|||
|
|
|
|||
|
|
// 将数据复制到WASM内存
|
|||
|
|
for (let i = 0; i < pcmData.length; i++) {
|
|||
|
|
mod.HEAP16[(pcmPtr >> 1) + i] = pcmData[i];
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 为Opus编码数据分配内存
|
|||
|
|
const maxEncodedSize = this.maxPacketSize;
|
|||
|
|
const encodedPtr = mod._malloc(maxEncodedSize);
|
|||
|
|
|
|||
|
|
// 编码
|
|||
|
|
const encodedBytes = mod._opus_encode(
|
|||
|
|
this.encoderPtr,
|
|||
|
|
pcmPtr,
|
|||
|
|
this.frameSize,
|
|||
|
|
encodedPtr,
|
|||
|
|
maxEncodedSize
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
if (encodedBytes < 0) {
|
|||
|
|
mod._free(pcmPtr);
|
|||
|
|
mod._free(encodedPtr);
|
|||
|
|
throw new Error(`Opus编码失败: ${encodedBytes}`);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 复制编码后的数据
|
|||
|
|
const encodedData = new Uint8Array(encodedBytes);
|
|||
|
|
for (let i = 0; i < encodedBytes; i++) {
|
|||
|
|
encodedData[i] = mod.HEAPU8[encodedPtr + i];
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 释放内存
|
|||
|
|
mod._free(pcmPtr);
|
|||
|
|
mod._free(encodedPtr);
|
|||
|
|
|
|||
|
|
return encodedData;
|
|||
|
|
},
|
|||
|
|
|
|||
|
|
// 销毁方法
|
|||
|
|
destroy: function() {
|
|||
|
|
if (this.encoderPtr) {
|
|||
|
|
this.module._free(this.encoderPtr);
|
|||
|
|
this.encoderPtr = null;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
// 创建解码器
|
|||
|
|
opusDecoder = {
|
|||
|
|
channels: CHANNELS,
|
|||
|
|
rate: SAMPLE_RATE,
|
|||
|
|
frameSize: FRAME_SIZE,
|
|||
|
|
module: mod,
|
|||
|
|
|
|||
|
|
// 初始化解码器
|
|||
|
|
init: function() {
|
|||
|
|
// 获取解码器大小
|
|||
|
|
const decoderSize = mod._opus_decoder_get_size(this.channels);
|
|||
|
|
console.log(`Opus解码器大小: ${decoderSize}字节`);
|
|||
|
|
|
|||
|
|
// 分配内存
|
|||
|
|
this.decoderPtr = mod._malloc(decoderSize);
|
|||
|
|
if (!this.decoderPtr) {
|
|||
|
|
throw new Error("无法分配解码器内存");
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 初始化解码器
|
|||
|
|
const err = mod._opus_decoder_init(
|
|||
|
|
this.decoderPtr,
|
|||
|
|
this.rate,
|
|||
|
|
this.channels
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
if (err < 0) {
|
|||
|
|
throw new Error(`Opus解码器初始化失败: ${err}`);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return true;
|
|||
|
|
},
|
|||
|
|
|
|||
|
|
// 解码方法
|
|||
|
|
decode: function(opusData) {
|
|||
|
|
const mod = this.module;
|
|||
|
|
|
|||
|
|
// 为Opus数据分配内存
|
|||
|
|
const opusPtr = mod._malloc(opusData.length);
|
|||
|
|
mod.HEAPU8.set(opusData, opusPtr);
|
|||
|
|
|
|||
|
|
// 为PCM输出分配内存
|
|||
|
|
const pcmPtr = mod._malloc(this.frameSize * 2); // Int16 = 2字节
|
|||
|
|
|
|||
|
|
// 解码
|
|||
|
|
const decodedSamples = mod._opus_decode(
|
|||
|
|
this.decoderPtr,
|
|||
|
|
opusPtr,
|
|||
|
|
opusData.length,
|
|||
|
|
pcmPtr,
|
|||
|
|
this.frameSize,
|
|||
|
|
0 // 不使用FEC
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
if (decodedSamples < 0) {
|
|||
|
|
mod._free(opusPtr);
|
|||
|
|
mod._free(pcmPtr);
|
|||
|
|
throw new Error(`Opus解码失败: ${decodedSamples}`);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 复制解码后的数据
|
|||
|
|
const decodedData = new Int16Array(decodedSamples);
|
|||
|
|
for (let i = 0; i < decodedSamples; i++) {
|
|||
|
|
decodedData[i] = mod.HEAP16[(pcmPtr >> 1) + i];
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 释放内存
|
|||
|
|
mod._free(opusPtr);
|
|||
|
|
mod._free(pcmPtr);
|
|||
|
|
|
|||
|
|
return decodedData;
|
|||
|
|
},
|
|||
|
|
|
|||
|
|
// 销毁方法
|
|||
|
|
destroy: function() {
|
|||
|
|
if (this.decoderPtr) {
|
|||
|
|
this.module._free(this.decoderPtr);
|
|||
|
|
this.decoderPtr = null;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
// 初始化编码器和解码器
|
|||
|
|
if (opusEncoder.init() && opusDecoder.init()) {
|
|||
|
|
console.log("Opus 编码器和解码器初始化成功。");
|
|||
|
|
return true;
|
|||
|
|
} else {
|
|||
|
|
console.error("Opus 初始化失败");
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
} catch (error) {
|
|||
|
|
console.error("Opus 初始化失败:", error);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 将Float32音频数据转换为Int16音频数据
|
|||
|
|
function convertFloat32ToInt16(float32Data) {
|
|||
|
|
const int16Data = new Int16Array(float32Data.length);
|
|||
|
|
for (let i = 0; i < float32Data.length; i++) {
|
|||
|
|
// 将[-1,1]范围转换为[-32768,32767]
|
|||
|
|
const s = Math.max(-1, Math.min(1, float32Data[i]));
|
|||
|
|
int16Data[i] = s < 0 ? s * 0x8000 : s * 0x7FFF;
|
|||
|
|
}
|
|||
|
|
return int16Data;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 将Int16音频数据转换为Float32音频数据
|
|||
|
|
function convertInt16ToFloat32(int16Data) {
|
|||
|
|
const float32Data = new Float32Array(int16Data.length);
|
|||
|
|
for (let i = 0; i < int16Data.length; i++) {
|
|||
|
|
// 将[-32768,32767]范围转换为[-1,1]
|
|||
|
|
float32Data[i] = int16Data[i] / (int16Data[i] < 0 ? 0x8000 : 0x7FFF);
|
|||
|
|
}
|
|||
|
|
return float32Data;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function startRecording() {
|
|||
|
|
if (isRecording) return;
|
|||
|
|
|
|||
|
|
// 确保有权限并且AudioContext是活跃的
|
|||
|
|
if (audioContext.state === 'suspended') {
|
|||
|
|
audioContext.resume().then(() => {
|
|||
|
|
console.log("AudioContext已恢复");
|
|||
|
|
continueStartRecording();
|
|||
|
|
}).catch(err => {
|
|||
|
|
console.error("恢复AudioContext失败:", err);
|
|||
|
|
statusLabel.textContent = "无法激活音频上下文，请再次点击";
|
|||
|
|
});
|
|||
|
|
} else {
|
|||
|
|
continueStartRecording();
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 实际开始录音的逻辑
|
|||
|
|
function continueStartRecording() {
|
|||
|
|
// 重置录音数据
|
|||
|
|
recordedPcmData = [];
|
|||
|
|
recordedOpusData = [];
|
|||
|
|
window.audioDataBuffer = new Int16Array(0); // 重置缓冲区
|
|||
|
|
|
|||
|
|
// 初始化Opus
|
|||
|
|
initOpus().then(success => {
|
|||
|
|
if (!success) {
|
|||
|
|
statusLabel.textContent = "Opus初始化失败";
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
console.log("开始录音，参数：", {
|
|||
|
|
sampleRate: SAMPLE_RATE,
|
|||
|
|
channels: CHANNELS,
|
|||
|
|
frameSize: FRAME_SIZE,
|
|||
|
|
bufferSize: BUFFER_SIZE
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
// 如果WebSocket已连接，发送开始录音信号
|
|||
|
|
if (isConnected && websocket && websocket.readyState === WebSocket.OPEN) {
|
|||
|
|
sendVoiceControlMessage('start');
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 请求麦克风权限
|
|||
|
|
navigator.mediaDevices.getUserMedia({
|
|||
|
|
audio: {
|
|||
|
|
sampleRate: SAMPLE_RATE,
|
|||
|
|
channelCount: CHANNELS,
|
|||
|
|
echoCancellation: true,
|
|||
|
|
noiseSuppression: true,
|
|||
|
|
autoGainControl: true
|
|||
|
|
}
|
|||
|
|
})
|
|||
|
|
.then(stream => {
|
|||
|
|
console.log("获取到麦克风流，实际参数：", stream.getAudioTracks()[0].getSettings());
|
|||
|
|
|
|||
|
|
// 检查流是否有效
|
|||
|
|
if (!stream || !stream.getAudioTracks().length || !stream.getAudioTracks()[0].enabled) {
|
|||
|
|
throw new Error("获取到的音频流无效");
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
mediaStream = stream;
|
|||
|
|
mediaSource = audioContext.createMediaStreamSource(stream);
|
|||
|
|
|
|||
|
|
// 创建ScriptProcessor(虽然已弃用，但兼容性好)
|
|||
|
|
// 在降级到ScriptProcessor之前尝试使用AudioWorklet
|
|||
|
|
createAudioProcessor().then(processor => {
|
|||
|
|
if (processor) {
|
|||
|
|
console.log("使用AudioWorklet处理音频");
|
|||
|
|
audioProcessor = processor;
|
|||
|
|
// 连接音频处理链
|
|||
|
|
mediaSource.connect(audioProcessor);
|
|||
|
|
audioProcessor.connect(audioContext.destination);
|
|||
|
|
} else {
|
|||
|
|
console.log("回退到ScriptProcessor");
|
|||
|
|
// 创建ScriptProcessor节点
|
|||
|
|
audioProcessor = audioContext.createScriptProcessor(BUFFER_SIZE, CHANNELS, CHANNELS);
|
|||
|
|
|
|||
|
|
// 处理音频数据
|
|||
|
|
audioProcessor.onaudioprocess = processAudioData;
|
|||
|
|
|
|||
|
|
// 连接音频处理链
|
|||
|
|
mediaSource.connect(audioProcessor);
|
|||
|
|
audioProcessor.connect(audioContext.destination);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 更新UI
|
|||
|
|
isRecording = true;
|
|||
|
|
statusLabel.textContent = "录音中...";
|
|||
|
|
startButton.disabled = true;
|
|||
|
|
stopButton.disabled = false;
|
|||
|
|
playButton.disabled = true;
|
|||
|
|
}).catch(error => {
|
|||
|
|
console.error("创建音频处理器失败:", error);
|
|||
|
|
statusLabel.textContent = "创建音频处理器失败";
|
|||
|
|
});
|
|||
|
|
})
|
|||
|
|
.catch(error => {
|
|||
|
|
console.error("获取麦克风失败:", error);
|
|||
|
|
statusLabel.textContent = "获取麦克风失败: " + error.message;
|
|||
|
|
});
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 创建AudioWorklet处理器
|
|||
|
|
async function createAudioProcessor() {
|
|||
|
|
try {
|
|||
|
|
// 尝试使用更现代的AudioWorklet API
|
|||
|
|
if ('AudioWorklet' in window && 'AudioWorkletNode' in window) {
|
|||
|
|
// 定义AudioWorklet处理器代码
|
|||
|
|
const workletCode = `
|
|||
|
|
class OpusRecorderProcessor extends AudioWorkletProcessor {
|
|||
|
|
constructor() {
|
|||
|
|
super();
|
|||
|
|
this.buffers = [];
|
|||
|
|
this.frameSize = ${FRAME_SIZE};
|
|||
|
|
this.buffer = new Float32Array(this.frameSize);
|
|||
|
|
this.bufferIndex = 0;
|
|||
|
|
this.isRecording = false;
|
|||
|
|
|
|||
|
|
this.port.onmessage = (event) => {
|
|||
|
|
if (event.data.command === 'start') {
|
|||
|
|
this.isRecording = true;
|
|||
|
|
} else if (event.data.command === 'stop') {
|
|||
|
|
this.isRecording = false;
|
|||
|
|
// 发送最后的缓冲区
|
|||
|
|
if (this.bufferIndex > 0) {
|
|||
|
|
const finalBuffer = this.buffer.slice(0, this.bufferIndex);
|
|||
|
|
this.port.postMessage({ buffer: finalBuffer });
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
process(inputs, outputs) {
|
|||
|
|
if (!this.isRecording) return true;
|
|||
|
|
|
|||
|
|
// 获取输入数据
|
|||
|
|
const input = inputs[0][0]; // mono channel
|
|||
|
|
if (!input || input.length === 0) return true;
|
|||
|
|
|
|||
|
|
// 将输入数据添加到缓冲区
|
|||
|
|
for (let i = 0; i < input.length; i++) {
|
|||
|
|
this.buffer[this.bufferIndex++] = input[i];
|
|||
|
|
|
|||
|
|
// 当缓冲区填满时，发送给主线程
|
|||
|
|
if (this.bufferIndex >= this.frameSize) {
|
|||
|
|
this.port.postMessage({ buffer: this.buffer.slice() });
|
|||
|
|
this.bufferIndex = 0;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
registerProcessor('opus-recorder-processor', OpusRecorderProcessor);
|
|||
|
|
`;
|
|||
|
|
|
|||
|
|
// 创建Blob URL
|
|||
|
|
const blob = new Blob([workletCode], { type: 'application/javascript' });
|
|||
|
|
const url = URL.createObjectURL(blob);
|
|||
|
|
|
|||
|
|
// 加载AudioWorklet模块
|
|||
|
|
await audioContext.audioWorklet.addModule(url);
|
|||
|
|
|
|||
|
|
// 创建AudioWorkletNode
|
|||
|
|
const workletNode = new AudioWorkletNode(audioContext, 'opus-recorder-processor');
|
|||
|
|
|
|||
|
|
// 处理从AudioWorklet接收的消息
|
|||
|
|
workletNode.port.onmessage = (event) => {
|
|||
|
|
if (event.data.buffer) {
|
|||
|
|
// 使用与ScriptProcessor相同的处理逻辑
|
|||
|
|
processAudioData({
|
|||
|
|
inputBuffer: {
|
|||
|
|
getChannelData: () => event.data.buffer
|
|||
|
|
}
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
// 启动录音
|
|||
|
|
workletNode.port.postMessage({ command: 'start' });
|
|||
|
|
|
|||
|
|
// 保存停止函数
|
|||
|
|
workletNode.stopRecording = () => {
|
|||
|
|
workletNode.port.postMessage({ command: 'stop' });
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
console.log("AudioWorklet 音频处理器创建成功");
|
|||
|
|
return workletNode;
|
|||
|
|
}
|
|||
|
|
} catch (error) {
|
|||
|
|
console.error("创建AudioWorklet失败，将使用ScriptProcessor:", error);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 如果AudioWorklet不可用或失败，返回null以便回退到ScriptProcessor
|
|||
|
|
return null;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 处理音频数据
|
|||
|
|
function processAudioData(e) {
|
|||
|
|
// 获取输入缓冲区
|
|||
|
|
const inputBuffer = e.inputBuffer;
|
|||
|
|
|
|||
|
|
// 获取第一个通道的Float32数据
|
|||
|
|
const inputData = inputBuffer.getChannelData(0);
|
|||
|
|
|
|||
|
|
// 添加调试信息
|
|||
|
|
const nonZeroCount = Array.from(inputData).filter(x => Math.abs(x) > 0.001).length;
|
|||
|
|
console.log(`接收到音频数据: ${inputData.length} 个样本, 非零样本数: ${nonZeroCount}`);
|
|||
|
|
|
|||
|
|
// 如果全是0，可能是麦克风没有正确获取声音
|
|||
|
|
if (nonZeroCount < 5) {
|
|||
|
|
console.warn("警告: 检测到大量静音样本，请检查麦克风是否正常工作");
|
|||
|
|
// 继续处理，以防有些样本确实是静音
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 存储PCM数据用于调试
|
|||
|
|
recordedPcmData.push(new Float32Array(inputData));
|
|||
|
|
|
|||
|
|
// 转换为Int16数据供Opus编码
|
|||
|
|
const int16Data = convertFloat32ToInt16(inputData);
|
|||
|
|
|
|||
|
|
// 如果收集到的数据不是FRAME_SIZE的整数倍，需要进行处理
|
|||
|
|
// 创建静态缓冲区来存储不足一帧的数据
|
|||
|
|
if (!window.audioDataBuffer) {
|
|||
|
|
window.audioDataBuffer = new Int16Array(0);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 合并之前缓存的数据和新数据
|
|||
|
|
const combinedData = new Int16Array(window.audioDataBuffer.length + int16Data.length);
|
|||
|
|
combinedData.set(window.audioDataBuffer);
|
|||
|
|
combinedData.set(int16Data, window.audioDataBuffer.length);
|
|||
|
|
|
|||
|
|
// 处理完整帧
|
|||
|
|
const frameCount = Math.floor(combinedData.length / FRAME_SIZE);
|
|||
|
|
console.log(`可编码的完整帧数: ${frameCount}, 缓冲区总大小: ${combinedData.length}`);
|
|||
|
|
|
|||
|
|
for (let i = 0; i < frameCount; i++) {
|
|||
|
|
const frameData = combinedData.subarray(i * FRAME_SIZE, (i + 1) * FRAME_SIZE);
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
console.log(`编码第 ${i+1}/${frameCount} 帧, 帧大小: ${frameData.length}`);
|
|||
|
|
const encodedData = opusEncoder.encode(frameData);
|
|||
|
|
if (encodedData) {
|
|||
|
|
console.log(`编码成功: ${encodedData.length} 字节`);
|
|||
|
|
recordedOpusData.push(encodedData);
|
|||
|
|
|
|||
|
|
// 如果WebSocket已连接，发送编码后的数据
|
|||
|
|
if (isConnected && websocket && websocket.readyState === WebSocket.OPEN) {
|
|||
|
|
sendOpusDataToServer(encodedData);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
} catch (error) {
|
|||
|
|
console.error(`Opus编码帧 ${i+1} 失败:`, error);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 保存剩余不足一帧的数据
|
|||
|
|
const remainingSamples = combinedData.length % FRAME_SIZE;
|
|||
|
|
if (remainingSamples > 0) {
|
|||
|
|
window.audioDataBuffer = combinedData.subarray(frameCount * FRAME_SIZE);
|
|||
|
|
console.log(`保留 ${remainingSamples} 个样本到下一次处理`);
|
|||
|
|
} else {
|
|||
|
|
window.audioDataBuffer = new Int16Array(0);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function stopRecording() {
|
|||
|
|
if (!isRecording) return;
|
|||
|
|
|
|||
|
|
// 处理剩余的缓冲数据
|
|||
|
|
if (window.audioDataBuffer && window.audioDataBuffer.length > 0) {
|
|||
|
|
console.log(`停止录音，处理剩余的 ${window.audioDataBuffer.length} 个样本`);
|
|||
|
|
// 如果剩余数据不足一帧，可以通过补零的方式凑成一帧
|
|||
|
|
if (window.audioDataBuffer.length < FRAME_SIZE) {
|
|||
|
|
const paddedFrame = new Int16Array(FRAME_SIZE);
|
|||
|
|
paddedFrame.set(window.audioDataBuffer);
|
|||
|
|
// 剩余部分填充为0
|
|||
|
|
for (let i = window.audioDataBuffer.length; i < FRAME_SIZE; i++) {
|
|||
|
|
paddedFrame[i] = 0;
|
|||
|
|
}
|
|||
|
|
try {
|
|||
|
|
console.log(`编码最后一帧(补零): ${paddedFrame.length} 样本`);
|
|||
|
|
const encodedData = opusEncoder.encode(paddedFrame);
|
|||
|
|
if (encodedData) {
|
|||
|
|
recordedOpusData.push(encodedData);
|
|||
|
|
|
|||
|
|
// 如果WebSocket已连接，发送最后一帧
|
|||
|
|
if (isConnected && websocket && websocket.readyState === WebSocket.OPEN) {
|
|||
|
|
sendOpusDataToServer(encodedData);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
} catch (error) {
|
|||
|
|
console.error("最后一帧Opus编码失败:", error);
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
// 如果数据超过一帧，按正常流程处理
|
|||
|
|
processAudioData({
|
|||
|
|
inputBuffer: {
|
|||
|
|
getChannelData: () => convertInt16ToFloat32(window.audioDataBuffer)
|
|||
|
|
}
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
window.audioDataBuffer = null;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 如果WebSocket已连接，发送停止录音信号
|
|||
|
|
if (isConnected && websocket && websocket.readyState === WebSocket.OPEN) {
|
|||
|
|
// 发送一个空帧作为结束标记
|
|||
|
|
const emptyFrame = new Uint8Array(0);
|
|||
|
|
websocket.send(emptyFrame);
|
|||
|
|
|
|||
|
|
// 发送停止录音控制消息
|
|||
|
|
sendVoiceControlMessage('stop');
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 如果使用的是AudioWorklet，调用其特定的停止方法
|
|||
|
|
if (audioProcessor && typeof audioProcessor.stopRecording === 'function') {
|
|||
|
|
audioProcessor.stopRecording();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 停止麦克风
|
|||
|
|
if (mediaStream) {
|
|||
|
|
mediaStream.getTracks().forEach(track => track.stop());
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 断开音频处理链
|
|||
|
|
if (audioProcessor) {
|
|||
|
|
try {
|
|||
|
|
audioProcessor.disconnect();
|
|||
|
|
if (mediaSource) mediaSource.disconnect();
|
|||
|
|
} catch (error) {
|
|||
|
|
console.warn("断开音频处理链时出错:", error);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 更新UI
|
|||
|
|
isRecording = false;
|
|||
|
|
statusLabel.textContent = "已停止录音，收集了 " + recordedOpusData.length + " 帧Opus数据";
|
|||
|
|
startButton.disabled = false;
|
|||
|
|
stopButton.disabled = true;
|
|||
|
|
playButton.disabled = recordedOpusData.length === 0;
|
|||
|
|
|
|||
|
|
console.log("录制完成:",
|
|||
|
|
"PCM帧数:", recordedPcmData.length,
|
|||
|
|
"Opus帧数:", recordedOpusData.length);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function playRecording() {
|
|||
|
|
if (!recordedOpusData.length) {
|
|||
|
|
statusLabel.textContent = "没有可播放的录音";
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 将所有Opus数据解码为PCM
|
|||
|
|
let allDecodedData = [];
|
|||
|
|
|
|||
|
|
for (const opusData of recordedOpusData) {
|
|||
|
|
try {
|
|||
|
|
// 解码为Int16数据
|
|||
|
|
const decodedData = opusDecoder.decode(opusData);
|
|||
|
|
|
|||
|
|
if (decodedData && decodedData.length > 0) {
|
|||
|
|
// 将Int16数据转换为Float32
|
|||
|
|
const float32Data = convertInt16ToFloat32(decodedData);
|
|||
|
|
|
|||
|
|
// 添加到总解码数据中
|
|||
|
|
allDecodedData.push(...float32Data);
|
|||
|
|
}
|
|||
|
|
} catch (error) {
|
|||
|
|
console.error("Opus解码失败:", error);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 如果没有解码出数据，返回
|
|||
|
|
if (allDecodedData.length === 0) {
|
|||
|
|
statusLabel.textContent = "解码失败，无法播放";
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 创建音频缓冲区
|
|||
|
|
const audioBuffer = audioContext.createBuffer(CHANNELS, allDecodedData.length, SAMPLE_RATE);
|
|||
|
|
audioBuffer.copyToChannel(new Float32Array(allDecodedData), 0);
|
|||
|
|
|
|||
|
|
// 创建音频源并播放
|
|||
|
|
const source = audioContext.createBufferSource();
|
|||
|
|
source.buffer = audioBuffer;
|
|||
|
|
source.connect(audioContext.destination);
|
|||
|
|
source.start();
|
|||
|
|
|
|||
|
|
// 更新UI
|
|||
|
|
statusLabel.textContent = "正在播放...";
|
|||
|
|
playButton.disabled = true;
|
|||
|
|
|
|||
|
|
// 播放结束后恢复UI
|
|||
|
|
source.onended = () => {
|
|||
|
|
statusLabel.textContent = "播放完毕";
|
|||
|
|
playButton.disabled = false;
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 处理二进制消息的修改版本
|
|||
|
|
async function handleBinaryMessage(data) {
|
|||
|
|
try {
|
|||
|
|
let arrayBuffer;
|
|||
|
|
|
|||
|
|
// 根据数据类型进行处理
|
|||
|
|
if (data instanceof ArrayBuffer) {
|
|||
|
|
arrayBuffer = data;
|
|||
|
|
console.log(`收到ArrayBuffer音频数据，大小: ${data.byteLength}字节`);
|
|||
|
|
} else if (data instanceof Blob) {
|
|||
|
|
// 如果是Blob类型，转换为ArrayBuffer
|
|||
|
|
arrayBuffer = await data.arrayBuffer();
|
|||
|
|
console.log(`收到Blob音频数据，大小: ${arrayBuffer.byteLength}字节`);
|
|||
|
|
} else {
|
|||
|
|
console.warn(`收到未知类型的二进制数据: ${typeof data}`);
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 创建Uint8Array用于处理
|
|||
|
|
const opusData = new Uint8Array(arrayBuffer);
|
|||
|
|
|
|||
|
|
if (opusData.length > 0) {
|
|||
|
|
// 将数据添加到缓冲队列
|
|||
|
|
audioBufferQueue.push(opusData);
|
|||
|
|
|
|||
|
|
// 如果收到的是第一个音频包，开始缓冲过程
|
|||
|
|
if (audioBufferQueue.length === 1 && !isAudioBuffering && !isAudioPlaying) {
|
|||
|
|
startAudioBuffering();
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
console.warn('收到空音频数据帧，可能是结束标志');
|
|||
|
|
|
|||
|
|
// 如果缓冲队列中有数据且没有在播放，立即开始播放
|
|||
|
|
if (audioBufferQueue.length > 0 && !isAudioPlaying) {
|
|||
|
|
playBufferedAudio();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 如果正在播放，发送结束信号
|
|||
|
|
if (isAudioPlaying && streamingContext) {
|
|||
|
|
streamingContext.endOfStream = true;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
} catch (error) {
|
|||
|
|
console.error(`处理二进制消息出错:`, error);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 开始音频缓冲过程
|
|||
|
|
function startAudioBuffering() {
|
|||
|
|
if (isAudioBuffering || isAudioPlaying) return;
|
|||
|
|
|
|||
|
|
isAudioBuffering = true;
|
|||
|
|
console.log("开始音频缓冲...");
|
|||
|
|
|
|||
|
|
// 设置超时，如果在一定时间内没有收集到足够的音频包，就开始播放
|
|||
|
|
setTimeout(() => {
|
|||
|
|
if (isAudioBuffering && audioBufferQueue.length > 0) {
|
|||
|
|
console.log(`缓冲超时，当前缓冲包数: ${audioBufferQueue.length}，开始播放`);
|
|||
|
|
playBufferedAudio();
|
|||
|
|
}
|
|||
|
|
}, 300); // 300ms超时
|
|||
|
|
|
|||
|
|
// 监控缓冲进度
|
|||
|
|
const bufferCheckInterval = setInterval(() => {
|
|||
|
|
if (!isAudioBuffering) {
|
|||
|
|
clearInterval(bufferCheckInterval);
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 当累积了足够的音频包，开始播放
|
|||
|
|
if (audioBufferQueue.length >= BUFFER_THRESHOLD) {
|
|||
|
|
clearInterval(bufferCheckInterval);
|
|||
|
|
console.log(`已缓冲 ${audioBufferQueue.length} 个音频包，开始播放`);
|
|||
|
|
playBufferedAudio();
|
|||
|
|
}
|
|||
|
|
}, 50);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 播放已缓冲的音频
|
|||
|
|
function playBufferedAudio() {
|
|||
|
|
if (isAudioPlaying || audioBufferQueue.length === 0) return;
|
|||
|
|
|
|||
|
|
isAudioPlaying = true;
|
|||
|
|
isAudioBuffering = false;
|
|||
|
|
|
|||
|
|
// 创建流式播放上下文
|
|||
|
|
if (!streamingContext) {
|
|||
|
|
streamingContext = {
|
|||
|
|
queue: [], // 已解码的PCM队列
|
|||
|
|
playing: false, // 是否正在播放
|
|||
|
|
endOfStream: false, // 是否收到结束信号
|
|||
|
|
source: null, // 当前音频源
|
|||
|
|
totalSamples: 0, // 累积的总样本数
|
|||
|
|
lastPlayTime: 0, // 上次播放的时间戳
|
|||
|
|
// 将Opus数据解码为PCM
|
|||
|
|
decodeOpusFrames: async function(opusFrames) {
|
|||
|
|
let decodedSamples = [];
|
|||
|
|
|
|||
|
|
for (const frame of opusFrames) {
|
|||
|
|
try {
|
|||
|
|
// 使用Opus解码器解码
|
|||
|
|
const frameData = opusDecoder.decode(frame);
|
|||
|
|
if (frameData && frameData.length > 0) {
|
|||
|
|
// 转换为Float32
|
|||
|
|
const floatData = convertInt16ToFloat32(frameData);
|
|||
|
|
decodedSamples.push(...floatData);
|
|||
|
|
}
|
|||
|
|
} catch (error) {
|
|||
|
|
console.error("Opus解码失败:", error);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (decodedSamples.length > 0) {
|
|||
|
|
// 添加到解码队列
|
|||
|
|
this.queue.push(...decodedSamples);
|
|||
|
|
this.totalSamples += decodedSamples.length;
|
|||
|
|
|
|||
|
|
// 如果累积了至少0.2秒的音频，开始播放
|
|||
|
|
const minSamples = SAMPLE_RATE * MIN_AUDIO_DURATION;
|
|||
|
|
if (!this.playing && this.queue.length >= minSamples) {
|
|||
|
|
this.startPlaying();
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
// 开始播放音频
|
|||
|
|
startPlaying: function() {
|
|||
|
|
if (this.playing || this.queue.length === 0) return;
|
|||
|
|
|
|||
|
|
this.playing = true;
|
|||
|
|
|
|||
|
|
// 创建新的音频缓冲区
|
|||
|
|
const minPlaySamples = Math.min(this.queue.length, SAMPLE_RATE); // 最多播放1秒
|
|||
|
|
const currentSamples = this.queue.splice(0, minPlaySamples);
|
|||
|
|
|
|||
|
|
const audioBuffer = audioContext.createBuffer(CHANNELS, currentSamples.length, SAMPLE_RATE);
|
|||
|
|
audioBuffer.copyToChannel(new Float32Array(currentSamples), 0);
|
|||
|
|
|
|||
|
|
// 创建音频源
|
|||
|
|
this.source = audioContext.createBufferSource();
|
|||
|
|
this.source.buffer = audioBuffer;
|
|||
|
|
|
|||
|
|
// 创建增益节点用于平滑过渡
|
|||
|
|
const gainNode = audioContext.createGain();
|
|||
|
|
|
|||
|
|
// 应用淡入淡出效果避免爆音
|
|||
|
|
const fadeDuration = 0.02; // 20毫秒
|
|||
|
|
gainNode.gain.setValueAtTime(0, audioContext.currentTime);
|
|||
|
|
gainNode.gain.linearRampToValueAtTime(1, audioContext.currentTime + fadeDuration);
|
|||
|
|
|
|||
|
|
const duration = audioBuffer.duration;
|
|||
|
|
if (duration > fadeDuration * 2) {
|
|||
|
|
gainNode.gain.setValueAtTime(1, audioContext.currentTime + duration - fadeDuration);
|
|||
|
|
gainNode.gain.linearRampToValueAtTime(0, audioContext.currentTime + duration);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 连接节点并开始播放
|
|||
|
|
this.source.connect(gainNode);
|
|||
|
|
gainNode.connect(audioContext.destination);
|
|||
|
|
|
|||
|
|
this.lastPlayTime = audioContext.currentTime;
|
|||
|
|
console.log(`开始播放 ${currentSamples.length} 个样本，约 ${(currentSamples.length / SAMPLE_RATE).toFixed(2)} 秒`);
|
|||
|
|
|
|||
|
|
// 播放结束后的处理
|
|||
|
|
this.source.onended = () => {
|
|||
|
|
this.source = null;
|
|||
|
|
this.playing = false;
|
|||
|
|
|
|||
|
|
// 如果队列中还有数据或者缓冲区有新数据，继续播放
|
|||
|
|
if (this.queue.length > 0) {
|
|||
|
|
setTimeout(() => this.startPlaying(), 10);
|
|||
|
|
} else if (audioBufferQueue.length > 0) {
|
|||
|
|
// 缓冲区有新数据，进行解码
|
|||
|
|
const frames = [...audioBufferQueue];
|
|||
|
|
audioBufferQueue = [];
|
|||
|
|
this.decodeOpusFrames(frames);
|
|||
|
|
} else if (this.endOfStream) {
|
|||
|
|
// 流已结束且没有更多数据
|
|||
|
|
console.log("音频播放完成");
|
|||
|
|
isAudioPlaying = false;
|
|||
|
|
streamingContext = null;
|
|||
|
|
} else {
|
|||
|
|
// 等待更多数据
|
|||
|
|
setTimeout(() => {
|
|||
|
|
// 如果仍然没有新数据，但有更多的包到达
|
|||
|
|
if (this.queue.length === 0 && audioBufferQueue.length > 0) {
|
|||
|
|
const frames = [...audioBufferQueue];
|
|||
|
|
audioBufferQueue = [];
|
|||
|
|
this.decodeOpusFrames(frames);
|
|||
|
|
} else if (this.queue.length === 0 && audioBufferQueue.length === 0) {
|
|||
|
|
// 真的没有更多数据了
|
|||
|
|
console.log("音频播放完成 (超时)");
|
|||
|
|
isAudioPlaying = false;
|
|||
|
|
streamingContext = null;
|
|||
|
|
}
|
|||
|
|
}, 500); // 500ms超时
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
this.source.start();
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 开始处理缓冲的数据
|
|||
|
|
const frames = [...audioBufferQueue];
|
|||
|
|
audioBufferQueue = []; // 清空缓冲队列
|
|||
|
|
|
|||
|
|
// 解码并播放
|
|||
|
|
streamingContext.decodeOpusFrames(frames);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 将旧的playOpusFromServer函数保留为备用方法
|
|||
|
|
function playOpusFromServerOld(opusData) {
|
|||
|
|
if (!opusDecoder) {
|
|||
|
|
initOpus().then(success => {
|
|||
|
|
if (success) {
|
|||
|
|
decodeAndPlayOpusDataOld(opusData);
|
|||
|
|
} else {
|
|||
|
|
statusLabel.textContent = "Opus解码器初始化失败";
|
|||
|
|
}
|
|||
|
|
});
|
|||
|
|
} else {
|
|||
|
|
decodeAndPlayOpusDataOld(opusData);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 旧的解码和播放函数作为备用
|
|||
|
|
function decodeAndPlayOpusDataOld(opusData) {
|
|||
|
|
let allDecodedData = [];
|
|||
|
|
|
|||
|
|
for (const frame of opusData) {
|
|||
|
|
try {
|
|||
|
|
const decodedData = opusDecoder.decode(frame);
|
|||
|
|
if (decodedData && decodedData.length > 0) {
|
|||
|
|
const float32Data = convertInt16ToFloat32(decodedData);
|
|||
|
|
allDecodedData.push(...float32Data);
|
|||
|
|
}
|
|||
|
|
} catch (error) {
|
|||
|
|
console.error("服务端Opus数据解码失败:", error);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (allDecodedData.length === 0) {
|
|||
|
|
statusLabel.textContent = "服务端数据解码失败";
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const audioBuffer = audioContext.createBuffer(CHANNELS, allDecodedData.length, SAMPLE_RATE);
|
|||
|
|
audioBuffer.copyToChannel(new Float32Array(allDecodedData), 0);
|
|||
|
|
|
|||
|
|
const source = audioContext.createBufferSource();
|
|||
|
|
source.buffer = audioBuffer;
|
|||
|
|
source.connect(audioContext.destination);
|
|||
|
|
source.start();
|
|||
|
|
|
|||
|
|
statusLabel.textContent = "正在播放服务端数据...";
|
|||
|
|
source.onended = () => statusLabel.textContent = "服务端数据播放完毕";
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 更新playOpusFromServer函数为Promise版本
|
|||
|
|
function playOpusFromServer(opusData) {
|
|||
|
|
// 为了兼容，我们将opusData添加到audioBufferQueue并触发播放
|
|||
|
|
if (Array.isArray(opusData) && opusData.length > 0) {
|
|||
|
|
for (const frame of opusData) {
|
|||
|
|
audioBufferQueue.push(frame);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 如果没有在播放和缓冲，启动流程
|
|||
|
|
if (!isAudioBuffering && !isAudioPlaying) {
|
|||
|
|
startAudioBuffering();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return new Promise(resolve => {
|
|||
|
|
// 我们无法准确知道何时播放完成，所以设置一个合理的超时
|
|||
|
|
setTimeout(resolve, 1000); // 1秒后认为已处理
|
|||
|
|
});
|
|||
|
|
} else {
|
|||
|
|
// 如果不是数组或为空，使用旧方法
|
|||
|
|
return new Promise(resolve => {
|
|||
|
|
playOpusFromServerOld(opusData);
|
|||
|
|
setTimeout(resolve, 1000);
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 连接WebSocket服务器
|
|||
|
|
function connectToServer() {
|
|||
|
|
let url = serverUrlInput.value || "ws://127.0.0.1:8000/xiaozhi/v1/";
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
// 检查URL格式
|
|||
|
|
if (!url.startsWith('ws://') && !url.startsWith('wss://')) {
|
|||
|
|
console.error('URL格式错误，必须以ws://或wss://开头');
|
|||
|
|
updateStatus('URL格式错误，必须以ws://或wss://开头', 'error');
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 添加认证参数
|
|||
|
|
let connUrl = new URL(url);
|
|||
|
|
connUrl.searchParams.append('device_id', 'web_test_device');
|
|||
|
|
connUrl.searchParams.append('device_mac', '00:11:22:33:44:55');
|
|||
|
|
|
|||
|
|
console.log(`正在连接: ${connUrl.toString()}`);
|
|||
|
|
updateStatus(`正在连接: ${connUrl.toString()}`, 'info');
|
|||
|
|
|
|||
|
|
websocket = new WebSocket(connUrl.toString());
|
|||
|
|
|
|||
|
|
// 设置接收二进制数据的类型为ArrayBuffer
|
|||
|
|
websocket.binaryType = 'arraybuffer';
|
|||
|
|
|
|||
|
|
websocket.onopen = async () => {
|
|||
|
|
console.log(`已连接到服务器: ${url}`);
|
|||
|
|
updateStatus(`已连接到服务器: ${url}`, 'success');
|
|||
|
|
isConnected = true;
|
|||
|
|
|
|||
|
|
// 连接成功后发送hello消息
|
|||
|
|
await sendHelloMessage();
|
|||
|
|
|
|||
|
|
if(connectButton.id === "connectButton") {
|
|||
|
|
connectButton.textContent = '断开';
|
|||
|
|
// connectButton.onclick = disconnectFromServer;
|
|||
|
|
connectButton.removeEventListener("click", connectToServer);
|
|||
|
|
connectButton.addEventListener("click", disconnectFromServer);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if(messageInput.id === "messageInput") {
|
|||
|
|
messageInput.disabled = false;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if(sendTextButton.id === "sendTextButton") {
|
|||
|
|
sendTextButton.disabled = false;
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
websocket.onclose = () => {
|
|||
|
|
console.log('已断开连接');
|
|||
|
|
updateStatus('已断开连接', 'info');
|
|||
|
|
isConnected = false;
|
|||
|
|
|
|||
|
|
if(connectButton.id === "connectButton") {
|
|||
|
|
connectButton.textContent = '连接';
|
|||
|
|
// connectButton.onclick = connectToServer;
|
|||
|
|
connectButton.removeEventListener("click", disconnectFromServer);
|
|||
|
|
connectButton.addEventListener("click", connectToServer);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if(messageInput.id === "messageInput") {
|
|||
|
|
messageInput.disabled = true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if(sendTextButton.id === "sendTextButton") {
|
|||
|
|
sendTextButton.disabled = true;
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
websocket.onerror = (error) => {
|
|||
|
|
console.error(`WebSocket错误:`, error);
|
|||
|
|
updateStatus(`WebSocket错误`, 'error');
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
websocket.onmessage = function (event) {
|
|||
|
|
try {
|
|||
|
|
// 检查是否为文本消息
|
|||
|
|
if (typeof event.data === 'string') {
|
|||
|
|
const message = JSON.parse(event.data);
|
|||
|
|
handleTextMessage(message);
|
|||
|
|
} else {
|
|||
|
|
// 处理二进制数据
|
|||
|
|
handleBinaryMessage(event.data);
|
|||
|
|
}
|
|||
|
|
} catch (error) {
|
|||
|
|
console.error(`WebSocket消息处理错误:`, error);
|
|||
|
|
// 非JSON格式文本消息直接显示
|
|||
|
|
if (typeof event.data === 'string') {
|
|||
|
|
addMessage(event.data);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
updateStatus('正在连接...', 'info');
|
|||
|
|
} catch (error) {
|
|||
|
|
console.error(`连接错误:`, error);
|
|||
|
|
updateStatus(`连接失败: ${error.message}`, 'error');
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 断开WebSocket连接
|
|||
|
|
function disconnectFromServer() {
|
|||
|
|
if (!websocket) return;
|
|||
|
|
|
|||
|
|
websocket.close();
|
|||
|
|
if (isRecording) {
|
|||
|
|
stopRecording();
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 发送hello握手消息
|
|||
|
|
async function sendHelloMessage() {
|
|||
|
|
if (!websocket || websocket.readyState !== WebSocket.OPEN) return;
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
// 设置设备信息
|
|||
|
|
const helloMessage = {
|
|||
|
|
type: 'hello',
|
|||
|
|
device_id: 'web_test_device',
|
|||
|
|
device_name: 'Web测试设备',
|
|||
|
|
device_mac: '00:11:22:33:44:55',
|
|||
|
|
token: 'your-token1' // 使用config.yaml中配置的token
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
console.log('发送hello握手消息');
|
|||
|
|
websocket.send(JSON.stringify(helloMessage));
|
|||
|
|
|
|||
|
|
// 等待服务器响应
|
|||
|
|
return new Promise(resolve => {
|
|||
|
|
// 5秒超时
|
|||
|
|
const timeout = setTimeout(() => {
|
|||
|
|
console.error('等待hello响应超时');
|
|||
|
|
resolve(false);
|
|||
|
|
}, 5000);
|
|||
|
|
|
|||
|
|
// 临时监听一次消息，接收hello响应
|
|||
|
|
const onMessageHandler = (event) => {
|
|||
|
|
try {
|
|||
|
|
const response = JSON.parse(event.data);
|
|||
|
|
if (response.type === 'hello' && response.session_id) {
|
|||
|
|
console.log(`服务器握手成功，会话ID: ${response.session_id}`);
|
|||
|
|
clearTimeout(timeout);
|
|||
|
|
websocket.removeEventListener('message', onMessageHandler);
|
|||
|
|
resolve(true);
|
|||
|
|
}
|
|||
|
|
} catch (e) {
|
|||
|
|
// 忽略非JSON消息
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
websocket.addEventListener('message', onMessageHandler);
|
|||
|
|
});
|
|||
|
|
} catch (error) {
|
|||
|
|
console.error(`发送hello消息错误:`, error);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 发送文本消息
|
|||
|
|
function sendTextMessage() {
|
|||
|
|
const message = messageInput ? messageInput.value.trim() : "";
|
|||
|
|
if (message === '' || !websocket || websocket.readyState !== WebSocket.OPEN) return;
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
// 发送listen消息
|
|||
|
|
const listenMessage = {
|
|||
|
|
type: 'listen',
|
|||
|
|
mode: 'manual',
|
|||
|
|
state: 'detect',
|
|||
|
|
text: message
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
websocket.send(JSON.stringify(listenMessage));
|
|||
|
|
addMessage(message, true);
|
|||
|
|
console.log(`发送文本消息: ${message}`);
|
|||
|
|
|
|||
|
|
if (messageInput) {
|
|||
|
|
messageInput.value = '';
|
|||
|
|
}
|
|||
|
|
} catch (error) {
|
|||
|
|
console.error(`发送消息错误:`, error);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 添加消息到会话记录
|
|||
|
|
function addMessage(text, isUser = false) {
|
|||
|
|
if (!conversationDiv) return;
|
|||
|
|
|
|||
|
|
const messageDiv = document.createElement('div');
|
|||
|
|
messageDiv.className = `message ${isUser ? 'user' : 'server'}`;
|
|||
|
|
messageDiv.textContent = text;
|
|||
|
|
conversationDiv.appendChild(messageDiv);
|
|||
|
|
conversationDiv.scrollTop = conversationDiv.scrollHeight;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 更新状态信息
|
|||
|
|
function updateStatus(message, type = 'info') {
|
|||
|
|
console.log(`[${type}] ${message}`);
|
|||
|
|
if (statusLabel) {
|
|||
|
|
statusLabel.textContent = message;
|
|||
|
|
}
|
|||
|
|
if (connectionStatus) {
|
|||
|
|
connectionStatus.textContent = message;
|
|||
|
|
switch(type) {
|
|||
|
|
case 'success':
|
|||
|
|
connectionStatus.style.color = 'green';
|
|||
|
|
break;
|
|||
|
|
case 'error':
|
|||
|
|
connectionStatus.style.color = 'red';
|
|||
|
|
break;
|
|||
|
|
case 'info':
|
|||
|
|
default:
|
|||
|
|
connectionStatus.style.color = 'black';
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 处理文本消息
|
|||
|
|
function handleTextMessage(message) {
|
|||
|
|
if (message.type === 'hello') {
|
|||
|
|
console.log(`服务器回应：${JSON.stringify(message, null, 2)}`);
|
|||
|
|
} else if (message.type === 'tts') {
|
|||
|
|
// TTS状态消息
|
|||
|
|
if (message.state === 'start') {
|
|||
|
|
console.log('服务器开始发送语音');
|
|||
|
|
} else if (message.state === 'sentence_start') {
|
|||
|
|
console.log(`服务器发送语音段: ${message.text}`);
|
|||
|
|
// 添加文本到会话记录
|
|||
|
|
if (message.text) {
|
|||
|
|
addMessage(message.text);
|
|||
|
|
}
|
|||
|
|
} else if (message.state === 'sentence_end') {
|
|||
|
|
console.log(`语音段结束: ${message.text}`);
|
|||
|
|
} else if (message.state === 'stop') {
|
|||
|
|
console.log('服务器语音传输结束');
|
|||
|
|
}
|
|||
|
|
} else if (message.type === 'audio') {
|
|||
|
|
// 音频控制消息
|
|||
|
|
console.log(`收到音频控制消息: ${JSON.stringify(message)}`);
|
|||
|
|
} else if (message.type === 'stt') {
|
|||
|
|
// 语音识别结果
|
|||
|
|
console.log(`识别结果: ${message.text}`);
|
|||
|
|
// 添加识别结果到会话记录
|
|||
|
|
addMessage(`[语音识别] ${message.text}`, true);
|
|||
|
|
} else if (message.type === 'llm') {
|
|||
|
|
// 大模型回复
|
|||
|
|
console.log(`大模型回复: ${message.text}`);
|
|||
|
|
// 添加大模型回复到会话记录
|
|||
|
|
if (message.text && message.text !== '😊') {
|
|||
|
|
addMessage(message.text);
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
// 未知消息类型
|
|||
|
|
console.log(`未知消息类型: ${message.type}`);
|
|||
|
|
addMessage(JSON.stringify(message, null, 2));
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 发送语音数据到WebSocket
|
|||
|
|
function sendOpusDataToServer(opusData) {
|
|||
|
|
if (!websocket || websocket.readyState !== WebSocket.OPEN) {
|
|||
|
|
console.error('WebSocket未连接，无法发送音频数据');
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
// 发送二进制数据
|
|||
|
|
websocket.send(opusData.buffer);
|
|||
|
|
console.log(`已发送Opus音频数据: ${opusData.length}字节`);
|
|||
|
|
return true;
|
|||
|
|
} catch (error) {
|
|||
|
|
console.error(`发送音频数据失败:`, error);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 发送语音开始和结束信号
|
|||
|
|
function sendVoiceControlMessage(state) {
|
|||
|
|
if (!websocket || websocket.readyState !== WebSocket.OPEN) return;
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
const message = {
|
|||
|
|
type: 'listen',
|
|||
|
|
mode: 'manual',
|
|||
|
|
state: state // 'start' 或 'stop'
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
websocket.send(JSON.stringify(message));
|
|||
|
|
console.log(`发送语音${state === 'start' ? '开始' : '结束'}控制消息`);
|
|||
|
|
} catch (error) {
|
|||
|
|
console.error(`发送语音控制消息失败:`, error);
|
|||
|
|
}
|
|||
|
|
}
|