From 5ce76141a36b3c3b051326c12cbd51568ca9ea2f Mon Sep 17 00:00:00 2001 From: Miroslav Pejic Date: Sat, 25 Sep 2021 19:48:27 +0200 Subject: [PATCH] [mirotalksfu] - add collaborative whiteboard --- README.md | 1 + public/Room.html | 33 ++++++ public/css/Room.css | 58 ++++++++-- public/js/Room.js | 232 +++++++++++++++++++++++++++++++++++++++- public/js/RoomClient.js | 18 +++- src/Server.js | 18 ++++ 6 files changed, 348 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index e595a19..4284416 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Powered by `WebRTC` with [SFU](https://mediasoup.org) integrated server. - Echo cancellation and noise suppression that makes your audio crystal clear - Screen Sharing to present documents, slides, and more ... - Chat with Emoji Picker to show you feeling and possibility to Save the conversations +- Collaborative whiteboard for the teachers - Select Microphone - Speaker and Video source - Recording your Screen, Audio or Video - Full Screen Mode on mouse click on the Video element diff --git a/public/Room.html b/public/Room.html index fb7f002..b31a3ba 100644 --- a/public/Room.html +++ b/public/Room.html @@ -79,6 +79,7 @@ + @@ -137,6 +138,22 @@ access to use this app. +
+ +
+
+ Line width + +
+ Line color +
+ +
+
+
+
@@ -163,6 +180,22 @@ access to use this app. + +
diff --git a/public/css/Room.css b/public/css/Room.css index 15fb679..b271249 100644 --- a/public/css/Room.css +++ b/public/css/Room.css @@ -38,6 +38,9 @@ --msger-height: 680px; --msger-width: 420px; --msger-bg: linear-gradient(to left, #1f1e1e, #000000); + --wb-width: 800px; + --wb-height: 600px; + --wb-bg: linear-gradient(to left, #1f1e1e, #000000); --left-msg-bg: #222328; --right-msg-bg: #0a0b0c; --box-shadow: 0px 8px 16px 0px rgba(0, 0, 0, 0.2); @@ -50,7 +53,7 @@ } html { - min-height: 100%; + height: 100%; } body { @@ -81,7 +84,7 @@ body { --------------------------------------------------------------*/ #openNavButton { - z-index: 2; + z-index: 3; position: absolute; cursor: pointer; padding: 10px; @@ -92,7 +95,7 @@ body { } .sidenav { - z-index: 3; + z-index: 4; height: 100%; width: 0; position: fixed; @@ -149,7 +152,7 @@ body { padding: 10px; top: 0; width: 250px; - background: black; + background: var(--body-bg); } #control button { @@ -162,6 +165,7 @@ body { #settings { position: relative; + background: var(--body-bg); } /*-------------------------------------------------------------- @@ -207,6 +211,7 @@ body { #recording { position: relative; + background: var(--body-bg); } #recording button, @@ -222,7 +227,7 @@ body { --------------------------------------------------------------*/ .chat-room { - z-index: 4; + z-index: 5; display: none; position: fixed; height: var(--msger-height); @@ -528,7 +533,7 @@ button:hover { --------------------------------------------------------------*/ #participants { - z-index: 5; + z-index: 6; position: absolute; margin: auto; padding: 10px; @@ -572,6 +577,37 @@ button:hover { } */ +/*-------------------------------------------------------------- +# Whiteboard +--------------------------------------------------------------*/ + +#whiteboardSettings { + position: relative; + background: var(--body-bg); +} + +#whiteboard { + z-index: 2; + position: absolute; + margin: auto; + padding: 10px; + width: var(--wb-width); + height: var(--wb-height); + background: var(--wb-bg); + border-radius: 10px; + overflow: hidden; +} + +.whiteboard-header { + display: flex; + justify-content: space-between; + background: rgb(0, 0, 0); + border-radius: 10px; + padding: 10px; + color: white; + cursor: move; +} + /*-------------------------------------------------------------- # Pulse class effect --------------------------------------------------------------*/ @@ -623,8 +659,10 @@ button:hover { /* z-index: - 1 videoMediaContainer - - 2 buttonBar - - 3 sidenav - - 4 chat - - 5 participants + - 2 whiteboard + - 3 buttonBar + - 4 sidenav + - 4 whiteboard + - 5 chat + - 6 participants */ diff --git a/public/js/Room.js b/public/js/Room.js index dcf806d..330d21c 100644 --- a/public/js/Room.js +++ b/public/js/Room.js @@ -41,6 +41,14 @@ let isVideoOn = true; let recTimer = null; let recElapsedTime = null; +const wbWidth = 800; +const wbHeight = 600; +let wbCanvas = null; +let wbIsDrawing = false; +let wbIsOpen = false; +var wbIsRedoing = false; +var wbPop = []; + const socket = io(); function getRandomNumber(length) { @@ -56,6 +64,11 @@ function getRandomNumber(length) { function initClient() { if (!DetectRTC.isMobileDevice) { setTippy('closeNavButton', 'Close', 'right'); + setTippy('whiteboardUndoBtn', 'Undo', 'top'); + setTippy('whiteboardRedoBtn', 'Redo', 'top'); + setTippy('whiteboardSaveBtn', 'Save', 'top'); + setTippy('whiteboardCleanBtn', 'Clear', 'top'); + setTippy('whiteboardCloseBtn', 'Close', 'top'); setTippy('participantsRefreshBtn', 'Refresh', 'top'); setTippy('participantsCloseBtn', 'Close', 'top'); setTippy('chatMessage', 'Press enter to send', 'top-start'); @@ -66,6 +79,7 @@ function initClient() { setTippy('chatCloseButton', 'Close', 'bottom'); setTippy('sessionTime', 'Session time', 'right'); } + setupWhiteboard(); initEnumerateDevices(); } @@ -394,6 +408,7 @@ function roomIsReady() { } else { rc.makeDraggable(chatRoom, chatHeader); rc.makeDraggable(participants, participantsHeader); + rc.makeDraggable(whiteboard, whiteboardHeader); if (navigator.getDisplayMedia || navigator.mediaDevices.getDisplayMedia) { show(startScreenButton); } @@ -408,6 +423,7 @@ function roomIsReady() { show(raiseHandButton); if (isAudioAllowed) show(startAudioButton); if (isVideoAllowed) show(startVideoButton); + show(whiteboardButton); show(participantsButton); show(lockRoomButton); show(aboutButton); @@ -579,6 +595,24 @@ function handleButtons() { stopScreenButton.onclick = () => { rc.closeProducer(RoomClient.mediaType.screen); }; + whiteboardButton.onclick = () => { + toggleWhiteboard(); + }; + whiteboardUndoBtn.onclick = () => { + whiteboardAction(getWhiteboardAction('undo')); + }; + whiteboardRedoBtn.onclick = () => { + whiteboardAction(getWhiteboardAction('redo')); + }; + whiteboardSaveBtn.onclick = () => { + wbCanvasSaveImg(); + }; + whiteboardCleanBtn.onclick = () => { + whiteboardAction(getWhiteboardAction('clear')); + }; + whiteboardCloseBtn.onclick = () => { + whiteboardAction(getWhiteboardAction('close')); + }; participantsButton.onclick = () => { getRoomParticipants(); }; @@ -604,6 +638,7 @@ function handleButtons() { // #################################################### function handleSelects() { + // devices options microphoneSelect.onchange = () => { rc.closeThenProduce(RoomClient.mediaType.audio, microphoneSelect.value); }; @@ -613,6 +648,13 @@ function handleSelects() { videoSelect.onchange = () => { rc.closeThenProduce(RoomClient.mediaType.video, videoSelect.value); }; + // whiteboard options + wbDrawingLineWidthEl.onchange = () => { + wbCanvas.freeDrawingBrush.width = parseInt(wbDrawingLineWidthEl.value, 10) || 1; + }; + wbDrawingColorEl.onchange = () => { + wbCanvas.freeDrawingBrush.color = wbDrawingColorEl.value; + }; } // #################################################### @@ -749,7 +791,7 @@ function handleRoomClientEvents() { } // #################################################### -// SHOW LOG +// UTILITY // #################################################### function userLog(icon, message, position, timer = 3000) { @@ -766,6 +808,26 @@ function userLog(icon, message, position, timer = 3000) { }); } +function saveDataToFile(dataURL, fileName) { + const a = document.createElement('a'); + a.style.display = 'none'; + a.href = dataURL; + a.download = fileName; + document.body.appendChild(a); + a.click(); + setTimeout(() => { + document.body.removeChild(a); + window.URL.revokeObjectURL(dataURL); + }, 100); +} + +function getDataTimeString() { + const d = new Date(); + const date = d.toISOString().split('T')[0]; + const time = d.toTimeString().split(' ')[0]; + return `${date}-${time}`; +} + // #################################################### // SOUND // #################################################### @@ -780,6 +842,174 @@ async function sound(name) { } } +// #################################################### +// HANDLE WHITEBOARD +// #################################################### + +function toggleWhiteboard() { + toggleWhiteboardSettings(); + let whiteboard = rc.getId('whiteboard'); + whiteboard.classList.toggle('show'); + whiteboard.style.top = '50%'; + whiteboard.style.left = '50%'; + wbIsOpen = wbIsOpen ? false : true; +} + +function toggleWhiteboardSettings() { + rc.getId('whiteboardSettings').classList.toggle('show'); +} + +function setupWhiteboard() { + setupWhiteboardCanvas(); + setupWhiteboardCanvasSize(); + setupWhiteboardLocalListners(); +} + +function setupWhiteboardCanvas() { + wbCanvas = new fabric.Canvas('wbCanvas'); + wbCanvas.freeDrawingBrush.color = '#FFFFFF'; + wbCanvas.freeDrawingBrush.width = 3; + wbCanvas.isDrawingMode = true; +} + +function setupWhiteboardCanvasSize() { + let optimalSize = [wbWidth, wbHeight]; + let scaleFactorX = window.innerWidth / optimalSize[0]; + let scaleFactorY = window.innerHeight / optimalSize[1]; + if (scaleFactorX < scaleFactorY && scaleFactorX < 1) { + wbCanvas.setWidth(optimalSize[0] * scaleFactorX); + wbCanvas.setHeight(optimalSize[1] * scaleFactorX); + wbCanvas.setZoom(scaleFactorX); + document.documentElement.style.setProperty('--wb-width', optimalSize[0] * scaleFactorX); + document.documentElement.style.setProperty('--wb-height', optimalSize[1] * scaleFactorX); + } else if (scaleFactorX > scaleFactorY && scaleFactorY < 1) { + wbCanvas.setWidth(optimalSize[0] * scaleFactorY); + wbCanvas.setHeight(optimalSize[1] * scaleFactorY); + wbCanvas.setZoom(scaleFactorY); + document.documentElement.style.setProperty('--wb-width', optimalSize[0] * scaleFactorY); + document.documentElement.style.setProperty('--wb-height', optimalSize[1] * scaleFactorY); + } else { + wbCanvas.setWidth(optimalSize[0]); + wbCanvas.setHeight(optimalSize[1]); + wbCanvas.setZoom(1); + document.documentElement.style.setProperty('--wb-width', optimalSize[0]); + document.documentElement.style.setProperty('--wb-height', optimalSize[1]); + } + wbCanvas.calcOffset(); + wbCanvas.renderAll(); +} + +function setupWhiteboardLocalListners() { + wbCanvas.on('mouse:down', function () { + mouseDown(); + }); + wbCanvas.on('mouse:up', function () { + mouseUp(); + }); + wbCanvas.on('mouse:move', function () { + mouseMove(); + }); + wbCanvas.on('object:added', function () { + objectAdded(); + }); +} + +function mouseDown() { + wbIsDrawing = true; +} + +function mouseUp() { + wbIsDrawing = false; + wbCanvasToJson(); +} + +function mouseMove() { + if (!wbIsDrawing) return; +} + +function objectAdded() { + if (!wbIsRedoing) wbPop = []; + wbIsRedoing = false; +} + +function wbCanvasUndo() { + if (wbCanvas._objects.length > 0) { + wbPop.push(wbCanvas._objects.pop()); + wbCanvas.renderAll(); + } +} + +function wbCanvasRedo() { + if (wbPop.length > 0) { + wbIsRedoing = true; + wbCanvas.add(wbPop.pop()); + } +} + +function wbCanvasSaveImg() { + const dataURL = wbCanvas.toDataURL({ + width: wbCanvas.getWidth(), + height: wbCanvas.getHeight(), + left: 0, + top: 0, + format: 'png', + }); + const dataNow = getDataTimeString(); + const fileName = `whiteboard-${dataNow}.png`; + saveDataToFile(dataURL, fileName); +} + +function wbCanvasToJson() { + if (rc.thereIsConsumers()) { + var wbCanvasJson = JSON.stringify(wbCanvas.toJSON()); + rc.socket.emit('wbCanvasToJson', wbCanvasJson); + } +} + +function JsonToWbCanvas(json) { + if (!wbIsOpen) toggleWhiteboard(); + + wbCanvas.loadFromJSON(json); + wbCanvas.renderAll(); +} + +function getWhiteboardAction(action) { + return { + peer_name: peer_name, + action: action, + }; +} + +function whiteboardAction(data, emit = true) { + if (emit) { + if (rc.thereIsConsumers()) { + rc.socket.emit('whiteboardAction', data); + } + } else { + userLog( + 'info', + `${data.peer_name} whiteboard action: ${data.action}`, + 'top-end', + ); + } + + switch (data.action) { + case 'undo': + wbCanvasUndo(); + break; + case 'redo': + wbCanvasRedo(); + break; + case 'clear': + wbCanvas.clear(); + break; + case 'close': + if (wbIsOpen) toggleWhiteboard(); + break; + //... + } +} + // #################################################### // HANDLE PARTICIPANTS // #################################################### diff --git a/public/js/RoomClient.js b/public/js/RoomClient.js index cbf049a..089be84 100644 --- a/public/js/RoomClient.js +++ b/public/js/RoomClient.js @@ -393,6 +393,22 @@ class RoomClient { }.bind(this), ); + this.socket.on( + 'wbCanvasToJson', + function (data) { + console.log('Received whiteboard canvas JSON'); + JsonToWbCanvas(data); + }.bind(this), + ); + + this.socket.on( + 'whiteboardAction', + function (data) { + console.log('Whiteboard action', data); + whiteboardAction(data, false); + }.bind(this), + ); + this.socket.on( 'disconnect', function () { @@ -1029,7 +1045,6 @@ class RoomClient { if (this.consumers.size > 0) { return true; } - this.userLog('info', 'No participants in the room', 'top-end'); return false; } @@ -1156,6 +1171,7 @@ class RoomClient { sendMessage() { if (!this.thereIsConsumers()) { chatMessage.value = ''; + this.userLog('info', 'No participants in the room', 'top-end'); return; } let peer_msg = this.formatMsg(chatMessage.value); diff --git a/src/Server.js b/src/Server.js index ff97ad9..d22b0e3 100644 --- a/src/Server.js +++ b/src/Server.js @@ -252,6 +252,17 @@ io.on('connection', (socket) => { roomList.get(socket.room_id).broadCast(socket.id, 'updatePeerInfo', data); }); + socket.on('wbCanvasToJson', (data) => { + // let objLength = bytesToSize(Object.keys(data).length); + // log.debug('Send Whiteboard canvas JSON', { length: objLength }); + roomList.get(socket.room_id).broadCast(socket.id, 'wbCanvasToJson', data); + }); + + socket.on('whiteboardAction', (data) => { + log.debug('Whiteboard', data); + roomList.get(socket.room_id).broadCast(socket.id, 'whiteboardAction', data); + }); + socket.on('join', (data, cb) => { if (!roomList.has(socket.room_id)) { return cb({ @@ -434,4 +445,11 @@ io.on('connection', (socket) => { roomList.get(socket.room_id) && roomList.get(socket.room_id).getPeers().get(socket.id).peer_info.peer_name ); } + + function bytesToSize(bytes) { + let sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB']; + if (bytes == 0) return '0 Byte'; + let i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024))); + return Math.round(bytes / Math.pow(1024, i), 2) + ' ' + sizes[i]; + } });