集成音视频 Realtime Web SDK

import { useRef, useState } from 'react';
import {
  EventNames,
  RealtimeAPIError,
  RealtimeClient,
  RealtimeError,
  RealtimeUtils,
} from '@coze/realtime-api';
import { Button, Space, List, message } from 'antd';
import { CozeAPI, COZE_CN_BASE_URL, ChatEventType } from '@coze/api';
import { useTokenWithPat } from './hooks';

// ⚠️ 替换成你的智能体ID
const botId = '在这里填入你的智能体ID';

function App() {
  // ... [这里的代码请参考：https://github.com/coze-dev/coze-js/blob/main/examples/realtime-quickstart-react/src/App.tsx ] ...
}

export default App;

import { useRef, useState } from 'react';
import {
  EventNames,
  RealtimeAPIError,
  RealtimeClient,
  RealtimeError,
  RealtimeUtils,
} from '@coze/realtime-api';
import { Button, Space, List, message } from 'antd';
import { CozeAPI, COZE_CN_BASE_URL, ChatEventType } from '@coze/api';

function App() {
  // 客户端引用
  const clientRef = useRef<RealtimeClient | null>(null);
  
  // 核心状态管理
  const [messageList, setMessageList] = useState<string[]>([]); // 消息列表
  const [isConnecting, setIsConnecting] = useState(false);     // 连接状态
  const [isConnected, setIsConnected] = useState(false);       // 已连接状态
  const [audioEnabled, setAudioEnabled] = useState(true);      // 麦克风状态
  const [isSupportVideo, setIsSupportVideo] = useState(false); // 视频支持状态
  // ...其它代码
}

  // 选择一种认证方式
  const { getToken } = useTokenWithPat();
  // 其他认证方式：
  // const { getToken } = useTokenWithDevice();
  // const { getToken } = useTokenWithJWT();
  // const { getToken } = useTokenWithPKCE();
  // const { getToken } = useTokenWithWeb();

  async function getVoices() {
    const api = new CozeAPI({
      token: getToken,
      baseURL: COZE_CN_BASE_URL,
      allowPersonalAccessTokenInBrowser: true,
    });
    const voices = await api.audio.voices.list();
    return voices.voice_list;
  }

  async function initClient() {
    // 检查设备权限
    const permission = await RealtimeUtils.checkDevicePermission(true);
    if (!permission.audio) {
      throw new Error('需要麦克风访问权限');
    }
    setIsSupportVideo(permission.video);

    // 获取音色列表
    const voices = await getVoices();

    // 初始化客户端
    const client = new RealtimeClient({
      accessToken: getToken,
      botId,
      connectorId: '1024',
      voiceId: voices.length > 0 ? voices[0].voice_id : undefined,
      allowPersonalAccessTokenInBrowser: true,
      debug: true,
      videoConfig: permission.video
        ? { renderDom: 'local-player' }
        : undefined,
    });

    clientRef.current = client;
    handleMessageEvent(); // 下面会实现
  }

  const handleMessageEvent = async () => {
    let lastEvent: any;

    clientRef.current?.on(EventNames.ALL_SERVER, (eventName, event: any) => {
      // 只处理消息增量更新和完成事件
      if (
        event.event_type !== ChatEventType.CONVERSATION_MESSAGE_DELTA &&
        event.event_type !== ChatEventType.CONVERSATION_MESSAGE_COMPLETED
      ) {
        return;
      }
      
      const content = event.data.content;
      setMessageList(prev => {
        // 处理增量更新
        if (lastEvent?.event_type === ChatEventType.CONVERSATION_MESSAGE_DELTA) {
          return [...prev.slice(0, -1), prev[prev.length - 1] + content];
        }
        // 处理新消息
        if (event.event_type === ChatEventType.CONVERSATION_MESSAGE_DELTA) {
          return [...prev, content];
        }
        return prev;
      });
      lastEvent = event;
    });
  };

  // 连接
  const handleConnect = async () => {
    try {
      if (!clientRef.current) {
        await initClient();
      }
      await clientRef.current?.connect();
      setIsConnected(true);
    } catch (error) {
      // 错误处理...
    }
  };

  // 打断
  const handleInterrupt = () => {
    try {
      clientRef.current?.interrupt();
    } catch (error) {
      message.error('打断失败：' + error);
    }
  };

  // 断开连接
  const handleDisconnect = () => {
    try {
      clientRef.current?.disconnect();
      clientRef.current?.clearEventHandlers();
      clientRef.current = null;
      setIsConnected(false);
    } catch (error) {
      message.error('断开失败：' + error);
    }
  };

  // 麦克风控制
  const toggleMicrophone = async () => {
    try {
      await clientRef.current?.setAudioEnable(!audioEnabled);
      setAudioEnabled(!audioEnabled);
    } catch (error) {
      message.error('切换麦克风状态失败：' + error);
    }
  };

  return (
    <div style={{ textAlign: 'center' }}>
      {/* 控制按钮 */}
      <Space style={{ padding: '20px' }}>
        <Button type="primary" onClick={handleConnect}>连接</Button>
        <Button onClick={handleInterrupt}>打断</Button>
        <Button danger onClick={handleDisconnect}>断开</Button>
        <Button onClick={toggleMicrophone}>
          {audioEnabled ? '静音' : '取消静音'}
        </Button>
      </Space>

      {/* 视频显示区域 */}
      {isSupportVideo && (
        <div id="local-player" style={{width: 400, height: 400}}></div>
      )}

      {/* 消息显示区域 */}
      <div style={{width: '400px', overflowY: 'auto'}}>
        <h3>实时语音回复</h3>
        <List
          dataSource={messageList}
          renderItem={(message, index) => (
            <List.Item key={index}>{message}</List.Item>
          )}
        />
      </div>
    </div>
  );

npm install @coze/realtime-api @coze/api

import { RealtimeUtils } from "@coze/realtime-api";

// 检查设备权限
const checkVideo = false; // 如需申请摄像头权限，请设置为 true
const result = await RealtimeUtils.checkDevicePermission();

// 检查麦克风权限
if (!result.audio) {
    throw new Error("需要麦克风访问权限");
}
// 如果申请摄像头权限，还需检查摄像头权限
if (checkVideo && !result.video) {
    throw new Error("需要摄像头访问权限");
}

import { RealtimeClient } from "@coze/realtime-api";

const client = new RealtimeClient({
    accessToken: 'your access token',   // 替换为准备工作中获取的访问密钥
    botId: 'your bot id',              // 替换为智能体 ID
    connectorId: '1024',              // 渠道 ID，固定为 1024
    allowPersonalAccessTokenInBrowser: true,  // 可选：允许在浏览器中使用个人访问令牌
});

import { EventNames } from "@coze/realtime-api";

// 监听所有事件
client.on(EventNames.ALL, (eventName, data) => {
    console.log(eventName, data);
});

await client.connect(); 

await client.disconnect(); 

import { EventNames, RealtimeClient, RealtimeUtils } from "@coze/realtime-api";

async function initClient() {
    try {
        // 1. 检查设备权限
        const result = await RealtimeUtils.checkDevicePermission();
        if (!result.audio) {
            throw new Error("需要麦克风访问权限");
        }

        // 2. 初始化客户端
        const client = new RealtimeClient({
            accessToken: 'your access token',
            botId: 'your bot id',
            connectorId: '1024',
            allowPersonalAccessTokenInBrowser: true,
        });

        // 3. 配置事件监听
        client.on(EventNames.ALL, (eventName, data) => {
            console.log(eventName, data);
        });

        // 4. 建立与服务端的连接
        await client.connect();
        
        // 返回初始化完成的客户端实例
        return client;
    } catch (error) {
        console.error('初始化失败:', error);
        throw error;
    }
}

interface RealtimeClientConfig {
  // 必填项
  accessToken: string;    // 访问令牌
  botId: string;         // Bot ID
  connectorId: string;   // 渠道 ID
  
  // 可选项
  voiceId?: string;      // 音色 ID
  conversationId?: string; // 会话 ID
  baseURL?: string;      // Base URL, 默认为 https://api.coze.cn
  userId?: string;        // 自定义用户Id
  workflowId?: string;    // 工作流Id
  debug?: boolean;       // 调试模式，建议开发模式下启用
  allowPersonalAccessTokenInBrowser?: boolean; // 允许在浏览器中使用个人访问令牌，不建议在生成环境下使用
  audioMutedDefault?: boolean; // 是否默认开启音频，默认 true
  suppressStationaryNoise?: boolean;    // 静态噪声抑制，默认 false
  suppressNonStationaryNoise?: boolean; // 非静态噪声抑制，默认 false
  videoConfig?: VideoConfig;    // 视频配置
}

// 使用示例
client.on(EventNames.ALL_SERVER, (eventName, event: any) => {
  if (eventName === 'server.bot.join') { // 这里需要加个 server. 前缀
    client.sendMessage({
      "id": "",
      "event_type": "conversation.message.create",
      "data": {
        "role": "user",
        "content_type": "text",
        "content": "你好"
      }
    });
  } else if (eventName === 'server.conversation.message.delta') {
    console.log('delta', event);
  }
});

// 事件监听示例 
client.on(EventNames.ALL, (eventName, data) => {
  console.log(`收到事件: ${eventName}`, data);
});

// 支持的事件类型
enum EventNames {
  ALL = 'realtime.event',           // 所有事件
  ALL_CLIENT = 'client.*',          // 所有客户端事件
  ALL_SERVER = 'server.*',          // 所有服务端事件
  CONNECTED = 'client.connected',    // 客户端已连接
  INTERRUPTED = 'client.interrupted', // 客户端已中断
  DISCONNECTED = 'client.disconnected', // 客户端已断开
  AUDIO_UNMUTED = 'client.audio.unmuted', // 音频已取消静音
  AUDIO_MUTED = 'client.audio.muted',     // 音频已静音
  ERROR = 'client.error'            // 客户端发生错误
}

enum RealtimeError { 
  DEVICE_ACCESS_ERROR = 'DEVICE_ACCESS_ERROR',         // 设备访问错误
  CONNECTION_ERROR = 'CONNECTION_ERROR',               // 连接错误
  DISCONNECTION_ERROR = 'DISCONNECTION_ERROR',         // 断开连接错误
  INTERRUPT_ERROR = 'INTERRUPT_ERROR',                 // 中断错误
  EVENT_HANDLER_ERROR = 'EVENT_HANDLER_ERROR',         // 事件处理错误
  NETWORK_ERROR = 'NETWORK_ERROR',                     // 网络错误
  CREATE_ROOM_ERROR = 'CREATE_ROOM_ERROR'             // 创建房间错误
}

import { AuthenticationError } from '@coze/api';
 
try {
  await client.connect();
} catch (error) {
      console.error(error);
      if (error instanceof RealtimeAPIError) {
        switch (error.code) {
          case RealtimeError.CREATE_ROOM_ERROR:
            console.log(`创建房间失败: ${error.message}`);
            break;
          case RealtimeError.CONNECTION_ERROR:
            console.log(`加入房间失败: ${error.message}`);
            break;
          case RealtimeError.DEVICE_ACCESS_ERROR:
            console.log(`获取设备失败: ${error.message}`);
            break;
          default:
            console.log(`发生错误: ${error.message}`);
        }
      } else {
        console.error('未知错误：', error);
      }
}