Teams虽然提供了转写的接口，但是不是实时的，即便使用订阅事件也不是实时的，为了达到实时转写的效果，使用recall.ai的转录和assembly_ai的转写实现。
前提：除Teams会议侧边栏应用开发-会议转写-CSDN博客的基本要求外，还需要修改用户的安全设置及设置Teams 工作账号，参考：Setup Guide (recall.ai)
一、服务端需要实现4个服务端点：
1）开始录音（创建机器人）
/*
* Send's a Recall Bot to start recording the call
*/
server.post('/start-recording', async (req, res) => {
const meeting_url = req.body.meetingUrl;
try {
if (!meeting_url) {
return res.status(400).json({ error: 'Missing meetingUrl' });
}
console.log('recall bot start recording', meeting_url);
const url = 'https://us-west-2.recall.ai/api/v1/bot/';
const options = {
method: 'POST',
headers: {
accept: 'application/json',
'content-type': 'application/json',
Authorization: `Token ${RECALL_API_KEY}`
},
body: JSON.stringify({
bot_name: 'teams bot',
real_time_transcription: {
destination_url: 'https://shortly-adapted-akita.ngrok-free.app/transcription?secret=' + WEBHOOK_SECRET,
partial_results: false
},
transcription_options: {provider: 'assembly_ai'},
meeting_url: meeting_url
})
};
const response = await fetch(url, options);
const bot = await response.json();
local_botId = bot.id
console.log('botId:', local_botId);
res.send(200, JSON.stringify({
botId: local_botId
}));
} catch (error) {
console.error("start-recoding error:", error);
}
});2）停止录音
/*
* Tells the Recall Bot to stop recording the call
*/
server.post('/stop-recording', async (req, res) => {
try {
const botId = local_botId;
if (!botId) {
res.send(400, JSON.stringify({ error: 'Missing botId' }));
}
await fetch(`https://us-west-2.recall.ai/api/v1/bot/${botId}/leave_call`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Accept: 'application/json',
Authorization: `Token ${RECALL_API_KEY}`
},
});
console.log('recall bot stopped');
res.send(200, {})
} catch (error) {
console.error("stop-recoding error:", error);
}
});
3）轮询机器人状态
/*
* Gets the current state of the Recall Bot
*/
server.get('/recording-state', async (req, res) => {
try {
const botId = local_botId;
if (!botId) {
res.send(400, JSON.stringify({ error: 'Missing botId' }));
}
const response = await fetch(`https://us-west-2.recall.ai/api/v1/bot/${botId}`, {
method: 'GET',
headers: {
'Content-Type': 'application/json',
Accept: 'application/json',
Authorization: `Token ${RECALL_API_KEY}`
},
});
const bot = await response.json();
const latestStatus = bot.status_changes.slice(-1)[0].code;
console.log('state:', latestStatus);
res.send(200, JSON.stringify({
state: latestStatus,
transcript: db.transcripts[botId] || [],
}));
} catch (error) {
console.error("recoding-state error:", error);
}
});
4）接收转写存储在db中（本例使用的是内存)
/*
* Receives transcription webhooks from the Recall Bot
*/
server.post('/transcription', async (req, res) => {
try {
console.log('transcription webhook received: ', req.body);
const { bot_id, transcript } = req.body.data;
if (!db.transcripts[bot_id]) {
db.transcripts[bot_id] = [];
}
if (transcript)
{
db.transcripts[bot_id].push(transcript);
}
res.send(200, JSON.stringify({ success: true }));
} catch (error) {
console.error("transcription error:", error);
}
});完整的服务端代码：
import restify from "restify";
import send from "send";
import fs from "fs";
import fetch from "node-fetch";
import path from 'path';
import { fileURLToPath } from 'url';
import { storeToken, getToken } from './redisClient.js';
import { WebSocketServer, WebSocket } from 'ws';
const __filename = fileURLToPath(import.meta.url);
console.log('__filename: ', __filename);
const __dirname = path.dirname(__filename);
console.log('__dirname: ', __dirname);
// Create HTTP server.
const server = restify.createServer({
key: process.env.SSL_KEY_FILE ? fs.readFileSync(process.env.SSL_KEY_FILE) : undefined,
certificate: process.env.SSL_CRT_FILE ? fs.readFileSync(process.env.SSL_CRT_FILE) : undefined,
formatters: {
"text/html": function (req, res, body) {
return body;
},
},
});
server.use(restify.plugins.bodyParser());
server.use(restify.plugins.queryParser());
server.get(
"/static/*",
restify.plugins.serveStatic({
directory: __dirname,
})
);
server.listen(process.env.port || process.env.PORT || 3000, function () {
console.log(`\n${server.name} listening to ${server.url}`);
});
// Adding tabs to our app. This will setup routes to various views
// Setup home page
server.get("/config", (req, res, next) => {
send(req, __dirname + "/config/config.html").pipe(res);
});
// Setup the static tab
server.get("/meetingTab", (req, res, next) => {
send(req, __dirname + "/panel/panel.html").pipe(res);
});
//获得用户token
server.get('/auth', (req, res, next) => {
res.status(200);
res.send(`
<!DOCTYPE html>
<html>
<head>
<script>
// Function to handle the token storage
async function handleToken() {
const hash = window.location.hash.substring(1);
const hashParams = new URLSearchParams(hash);
const access_token = hashParams.get('access_token');
console.log('Received hash parameters:', hashParams);
if (access_token) {
console.log('Access token found:', access_token);
localStorage.setItem("access_token", access_token);
console.log('Access token stored in localStorage');
try {
const response = await fetch('https://shortly-adapted-akita.ngrok-free.app/store_user_token', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({ "user_token" : access_token })
});
if (response.ok) {
console.log('Token stored successfully');
} else {
console.error('Failed to store token:', response.statusText);
}
} catch (error) {
console.error('Error storing token:', error);
}
} else {
console.log('No access token found');
}
window.close();
}
// Call the function to handle the token
handleToken();
</script>
</head>
<body></body>
</html>
`);
next();
});
// 存储 user_token
server.post('/store_user_token', async (req, res) => {
const user_token = req.body.user_token;
if (!user_token) {
res.status(400);
res.send('user_token are required');
}
try {
// Store user token
await storeToken('user_token', user_token);
console.log('user_token stored in Redis');
} catch (err) {
console.error('user_token store Error:', err);
}
res.status(200);
res.send('Token stored successfully');
});
// 获取 user_token
server.get('/get_user_token', async (req, res) => {
try {
// Store user token
const user_token = await getToken('user_token');
console.log('user_token get in Redis');
res.send({"user_token": user_token});
} catch (err) {
console.error('user_token get Error:', err);
}
});
//应用token
let app_token = '';
const app_token_refresh_interval = 3000 * 1000; // 3000秒
const getAppToken = async () => {
try {
// 构建请求体
const requestBody = new URLSearchParams({
"grant_type": "client_credentials",
"client_id": "Azure注册应用ID",
"client_secret": "Azure注册应用密钥",
"scope": "https://graph.microsoft.com/.default",
}).toString();
// 获取app令牌
const tokenUrl = `https://login.microsoftonline.com/864168b4-813c-411a-827a-af408f70c665/oauth2/v2.0/token`;
const tokenResponse = await fetch(tokenUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
},
body: requestBody,
});
if (!tokenResponse.ok) {
const errorData = await tokenResponse.json();
throw new Error(errorData.error_description);
}
const tokenData = await tokenResponse.json();
app_token = tokenData.access_token;
console.log("app_token received!");
} catch (error) {
console.error('Error getting app token:', error);
}
};
// 定期刷新 app_token
setInterval(getAppToken, app_token_refresh_interval);
// 确保在服务器启动时获取 app_token
getAppToken();
//存储机器人转写信息
const db = {
transcripts: {
// [bot id]: [transcript]
},
};
const RECALL_API_KEY = '你的recall.ai的API KEY';
const WEBHOOK_SECRET = '在recall.ai配置webhook端点时的密钥';
let local_botId = null;
/*
* Send's a Recall Bot to start recording the call
*/
server.post('/start-recording', async (req, res) => {
const meeting_url = req.body.meetingUrl;
try {
if (!meeting_url) {
return res.status(400).json({ error: 'Missing meetingUrl' });
}
console.log('recall bot start recording', meeting_url);
const url = 'https://us-west-2.recall.ai/api/v1/bot/';
const options = {
method: 'POST',
headers: {
accept: 'application/json',
'content-type': 'application/json',
Authorization: `Token ${RECALL_API_KEY}`
},
body: JSON.stringify({
bot_name: 'teams bot',
real_time_transcription: {
destination_url: 'https://shortly-adapted-akita.ngrok-free.app/transcription?secret=' + WEBHOOK_SECRET,
partial_results: false
},
transcription_options: {provider: 'assembly_ai'},
meeting_url: meeting_url
})
};
const response = await fetch(url, options);
const bot = await response.json();
local_botId = bot.id
console.log('botId:', local_botId);
res.send(200, JSON.stringify({
botId: local_botId
}));
} catch (error) {
console.error("start-recoding error:", error);
}
});
/*
* Tells the Recall Bot to stop recording the call
*/
server.post('/stop-recording', async (req, res) => {
try {
const botId = local_botId;
if (!botId) {
res.send(400, JSON.stringify({ error: 'Missing botId' }));
}
await fetch(`https://us-west-2.recall.ai/api/v1/bot/${botId}/leave_call`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Accept: 'application/json',
Authorization: `Token ${RECALL_API_KEY}`
},
});
console.log('recall bot stopped');
res.send(200, {})
} catch (error) {
console.error("stop-recoding error:", error);
}
});
/*
* Gets the current state of the Recall Bot
*/
server.get('/recording-state', async (req, res) => {
try {
const botId = local_botId;
if (!botId) {
res.send(400, JSON.stringify({ error: 'Missing botId' }));
}
const response = await fetch(`https://us-west-2.recall.ai/api/v1/bot/${botId}`, {
method: 'GET',
headers: {
'Content-Type': 'application/json',
Accept: 'application/json',
Authorization: `Token ${RECALL_API_KEY}`
},
});
const bot = await response.json();
const latestStatus = bot.status_changes.slice(-1)[0].code;
console.log('state:', latestStatus);
res.send(200, JSON.stringify({
state: latestStatus,
transcript: db.transcripts[botId] || [],
}));
} catch (error) {
console.error("recoding-state error:", error);
}
});
/*
* Receives transcription webhooks from the Recall Bot
*/
server.post('/transcription', async (req, res) => {
try {
console.log('transcription webhook received: ', req.body);
const { bot_id, transcript } = req.body.data;
if (!db.transcripts[bot_id]) {
db.transcripts[bot_id] = [];
}
if (transcript)
{
db.transcripts[bot_id].push(transcript);
}
res.send(200, JSON.stringify({ success: true }));
} catch (error) {
console.error("transcription error:", error);
}
});二、页面需要实现开始录音和停止录音按钮及转写显示。
完整的页面代码：
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Meeting Transcripts</title>
<script src="https://res.cdn.office.net/teams-js/2.0.0/js/MicrosoftTeams.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.0.1/socket.io.js"></script>
<style>
.subtitle {
display: flex;
align-items: center;
margin-bottom: 10px;
}
.speaker-photo {
width: 20px;
height: 20px;
border-radius: 50%;
margin-right: 10px;
}
button {
padding: 5px 10px; /* 调整按钮的 padding 以减小高度 */
font-size: 14px; /* 调整按钮的字体大小 */
margin-right: 10px;
}
#transcript {
margin-top: 20px;
padding: 10px;
border: 1px solid #ccc;
min-height: 100px;
width: 100%;
}
</style>
</head>
<body>
<h2>Meeting Transcripts</h2>
<button id="startRecording">Start Recording</button>
<button id="stopRecording" disabled>Stop Recording</button>
<div id="transcripts"></div>
<script>
const clientId = 'Azure注册应用ID';
const tenantId = 'Azure注册应用租户ID';
const authUrl = `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/authorize`;
const redirectUri = 'https://shortly-adapted-akita.ngrok-free.app/auth'; // 确保与服务器端一致
const scope = 'user.read';
let user_token = null;
let meetingOrganizerUserId = null;
let participants = {}; // 用于存储参会者的信息
let userPhotoCache = {}; // 用于缓存用户头像
let tokenFetched = false; // 标志变量，用于跟踪是否已经获取了 user_token
let displayedTranscriptIds = new Set(); // 用于存储已经显示的转录片段的 ID
const getUserInfo = async (userId, accessToken) => {
const graphUrl = `https://graph.microsoft.com/v1.0/users/${userId}`;
const response = await fetch(graphUrl, {
headers: {
'Authorization': `Bearer ${accessToken}`
}
});
if (response.status === 401) {
// 如果 token 超期，重新触发 initAuthentication
initAuthentication();
return null;
}
const userInfo = await response.json();
return userInfo;
};
const getUserPhoto = async (userId, accessToken) => {
if (userPhotoCache[userId]) {
return userPhotoCache[userId];
}
const graphUrl = `https://graph.microsoft.com/v1.0/users/${userId}/photo/$value`;
const response = await fetch(graphUrl, {
headers: {
'Authorization': `Bearer ${accessToken}`
}
});
if (!response.ok) {
const errorData = await response.json();
console.error('Error fetching user photo:', errorData);
return null;
}
const photoBlob = await response.blob();
const photoUrl = URL.createObjectURL(photoBlob);
userPhotoCache[userId] = photoUrl; // 缓存头像 URL
return photoUrl;
};
const getMeetingDetails = async (user_token, joinMeetingId) => {
const apiUrl = `https://graph.microsoft.com/v1.0/me/onlineMeetings?$filter=joinMeetingIdSettings/joinMeetingId eq '${joinMeetingId}'`;
const response = await fetch(apiUrl, {
method: 'GET',
headers: {
'Authorization': `Bearer ${user_token}`,
'Content-Type': 'application/json'
}
});
if (!response.ok) {
const errorData = await response.json();
throw new Error(`getMeetingDetails status: ${response.status}, message: ${errorData.error}`);
}
const data = await response.json();
return data.value[0];
};
const getTranscriptContent = async (transcripts) => {
const subtitles = [];
try {
transcripts.forEach(transcript => {
const startTime = transcript.words[0].start_time;
const endTime = transcript.words[transcript.words.length - 1].end_time;
const speaker = transcript.speaker;
const content = transcript.words.map(word => word.text).join(' ');
subtitles.push({ startTime, endTime, speaker, content, id: transcript.original_transcript_id });
});
return subtitles;
} catch (error) {
console.error('getTranscriptContent error:', error);
return subtitles;
}
};
const displaySubtitle = async (subtitle, transcriptElement, accessToken) => {
const subtitleElement = document.createElement('div');
subtitleElement.classList.add('subtitle');
// 获取说话者的头像
const speakerUserId = participants[subtitle.speaker];
const speakerPhotoUrl = speakerUserId ? await getUserPhoto(speakerUserId, accessToken) : 'default-avatar.png';
// 创建头像元素
const speakerPhotoElement = document.createElement('img');
speakerPhotoElement.src = speakerPhotoUrl;
speakerPhotoElement.alt = subtitle.speaker;
speakerPhotoElement.classList.add('speaker-photo');
// 创建输出字符串
const output = `${subtitle.startTime} - ${subtitle.endTime}\n${subtitle.content}`;
subtitleElement.appendChild(speakerPhotoElement);
subtitleElement.appendChild(document.createTextNode(output));
transcriptElement.appendChild(subtitleElement);
};
const init = async () => {
try {
if (!tokenFetched) {
const response = await fetch('https://shortly-adapted-akita.ngrok-free.app/get_user_token');
const data = await response.json();
if (response.ok) {
user_token = data.user_token;
console.log('user token retrieved:', user_token);
tokenFetched = true;
} else {
console.error('Failed to get token:', response.statusText);
return;
}
}
console.log('User Token:', user_token);
const joinMeetingId = '45756456529'; // 替换为你要查询的 joinMeetingId
try {
const meetingDetails = await getMeetingDetails(user_token, joinMeetingId);
console.log('Meeting Details:', meetingDetails);
meetingOrganizerUserId = meetingDetails.participants.organizer.identity.user.id;
const meetingId = meetingDetails.id; // 获取会议 ID
console.log('Organizer User ID:', meetingOrganizerUserId);
console.log('Meeting ID:', meetingId);
// 获取主持人信息
const organizerInfo = await getUserInfo(meetingOrganizerUserId, user_token);
const organizerDisplayName = organizerInfo.displayName;
participants[organizerDisplayName] = meetingOrganizerUserId;
// 获取参会者信息
const attendeesPromises = meetingDetails.participants.attendees.map(async attendee => {
const userId = attendee.identity.user.id;
const userInfo = await getUserInfo(userId, user_token);
const displayName = userInfo.displayName;
participants[displayName] = userId;
});
await Promise.all(attendeesPromises);
} catch (error) {
console.error('Error fetching meeting details:', error);
}
} catch (error) {
console.error('Error getting token:', error);
}
};
const initAuthentication = () => {
microsoftTeams.app.initialize();
microsoftTeams.authentication.authenticate({
url: `${authUrl}?client_id=${clientId}&response_type=token&redirect_uri=${encodeURIComponent(redirectUri)}&scope=${encodeURIComponent(scope)}`,
width: 600,
height: 535,
successCallback: async (result) => {
console.log('Authentication success:', result);
},
failureCallback: (error) => {
console.error('Authentication failed:', error);
}
});
};
// 设置较长的轮询时间来防止 user_token 的超期
setInterval(initAuthentication, 3000000); // 每3000秒（50分钟）轮询一次
initAuthentication();
init();
// 录音控制功能
const startRecordingButton = document.getElementById('startRecording');
const stopRecordingButton = document.getElementById('stopRecording');
const transcriptDiv = document.getElementById('transcript');
let recordingInterval;
// Function to start recording
async function startRecording() {
const meetingUrl = await getMeetingUrl();
if (!meetingUrl) return;
try {
const response = await fetch('/start-recording', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ meetingUrl }),
});
if (response.ok) {
const data = await response.json();
console.log('Bot started:', data);
startRecordingButton.disabled = true;
stopRecordingButton.disabled = false;
startPolling();
} else {
console.error('Failed to start recording:', response.statusText);
}
} catch (error) {
console.error('Error starting recording:', error);
}
}
// Function to stop recording
async function stopRecording() {
try {
const response = await fetch('/stop-recording', {
method: 'POST',
});
if (response.ok) {
console.log('Bot stopped');
startRecordingButton.disabled = false;
stopRecordingButton.disabled = true;
clearInterval(recordingInterval);
} else {
console.error('Failed to stop recording:', response.statusText);
}
} catch (error) {
console.error('Error stopping recording:', error);
}
}
// Function to poll the recording state
async function pollRecordingState() {
try {
const response = await fetch('/recording-state');
if (response.ok) {
const data = await response.json();
updateUI(data);
} else {
console.error('Failed to get recording state:', response.statusText);
}
} catch (error) {
console.error('Error polling recording state:', error);
}
}
// Function to update the UI based on the recording state
function updateUI(data) {
const { state, transcript } = data;
console.log(state, transcript);
// Update the transcript display
const transcriptsContainer = document.getElementById('transcripts');
const transcriptElement = document.createDocumentFragment(); // 使用 DocumentFragment 优化 DOM 操作
if (transcript.length > 0) {
getTranscriptContent(transcript)
.then(subtitles => {
subtitles.forEach(subtitle => {
if (!displayedTranscriptIds.has(subtitle.id)) {
displaySubtitle(subtitle, transcriptElement, user_token);
displayedTranscriptIds.add(subtitle.id); // 添加到已显示的转录片段 ID 集合中
}
});
})
.catch(error => {
const errorElement = document.createElement('div');
errorElement.innerHTML = `<strong>${error}</strong>`;
transcriptElement.appendChild(errorElement);
})
.finally(() => {
transcriptsContainer.appendChild(transcriptElement); // 一次性插入 DOM
});
}
// Update button states based on the recording state
if (state === 'recording') {
startRecordingButton.disabled = true;
stopRecordingButton.disabled = false;
} else if (state === 'stopped') {
startRecordingButton.disabled = false;
stopRecordingButton.disabled = true;
}
}
// Function to start polling the recording state every 2 seconds
function startPolling() {
recordingInterval = setInterval(pollRecordingState, 2000);
}
// Event listeners for buttons
startRecordingButton.addEventListener('click', startRecording);
stopRecordingButton.addEventListener('click', stopRecording);
// Function to get the meeting URL from the meeting details
async function getMeetingUrl() {
const joinMeetingId = '45756456529'; // 替换为你要查询的 joinMeetingId
try {
const meetingDetails = await getMeetingDetails(user_token, joinMeetingId);
return meetingDetails.joinWebUrl;
} catch (error) {
console.error('Error fetching meeting URL:', error);
return null;
}
}
</script>
</body>
</html>最终效果：
![[论文笔记] Chain-of-Thought Reasoning without Prompting](https://i-blog.csdnimg.cn/direct/eb83f29e105e45769e97d9a9918fdc44.png)
